| Message ID | 20220105225013.1567871-6-lizhi.hou@xilinx.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | XRT Alveo driver infrastructure overview | expand |
On 1/5/22 2:50 PM, Lizhi Hou wrote: > The PCIE device driver which attaches to management function on Alveo > devices. It instantiates one or more partition. Each partition consists > a set of hardward endpoints. A flat device tree is associated with each > partition. The first version of this driver uses test version flat device > tree and call xrt lib API to unflatten it. > > Signed-off-by: Sonal Santan <sonal.santan@xilinx.com> > Signed-off-by: Max Zhen <max.zhen@xilinx.com> > Signed-off-by: Lizhi Hou <lizhi.hou@xilinx.com> > --- > drivers/fpga/Makefile | 1 + > drivers/fpga/xrt/Kconfig | 1 + > drivers/fpga/xrt/mgmt/Kconfig | 14 +++ > drivers/fpga/xrt/mgmt/Makefile | 16 +++ > drivers/fpga/xrt/mgmt/dt-test.dts | 12 +++ > drivers/fpga/xrt/mgmt/dt-test.h | 15 +++ > drivers/fpga/xrt/mgmt/xmgmt-drv.c | 158 ++++++++++++++++++++++++++++++ > 7 files changed, 217 insertions(+) > create mode 100644 drivers/fpga/xrt/mgmt/Kconfig > create mode 100644 drivers/fpga/xrt/mgmt/Makefile > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.dts > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.h > create mode 100644 drivers/fpga/xrt/mgmt/xmgmt-drv.c > > diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile > index 5bd41cf4c7ec..544e2144878f 100644 > --- a/drivers/fpga/Makefile > +++ b/drivers/fpga/Makefile > @@ -52,3 +52,4 @@ obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o > > # XRT drivers for Alveo > obj-$(CONFIG_FPGA_XRT_LIB) += xrt/lib/ > +obj-$(CONFIG_FPGA_XRT_XMGMT) += xrt/mgmt/ > diff --git a/drivers/fpga/xrt/Kconfig b/drivers/fpga/xrt/Kconfig > index 04c3bb5aaf4f..50422f77c6df 100644 > --- a/drivers/fpga/xrt/Kconfig > +++ b/drivers/fpga/xrt/Kconfig > @@ -4,3 +4,4 @@ > # > > source "drivers/fpga/xrt/lib/Kconfig" > +source "drivers/fpga/xrt/mgmt/Kconfig" > diff --git a/drivers/fpga/xrt/mgmt/Kconfig b/drivers/fpga/xrt/mgmt/Kconfig > new file mode 100644 > index 000000000000..a978747482be > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/Kconfig > @@ -0,0 +1,14 @@ > +# SPDX-License-Identifier: GPL-2.0-only > +# > +# Xilinx XRT FPGA device configuration > +# > + > +config FPGA_XRT_XMGMT > + tristate "Xilinx Alveo Management Driver" > + depends on FPGA_XRT_LIB > + select FPGA_BRIDGE > + select FPGA_REGION > + help > + Select this option to enable XRT PCIe driver for Xilinx Alveo FPGA. > + This driver provides interfaces for userspace application to access > + Alveo FPGA device. > diff --git a/drivers/fpga/xrt/mgmt/Makefile b/drivers/fpga/xrt/mgmt/Makefile > new file mode 100644 > index 000000000000..c5134bf71cca > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/Makefile > @@ -0,0 +1,16 @@ > +# SPDX-License-Identifier: GPL-2.0 > +# > +# Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved. > +# > +# Authors: Sonal.Santan@xilinx.com > +# > + > +FULL_XRT_PATH=$(srctree)/$(src)/.. > + > +obj-$(CONFIG_FPGA_XRT_LIB) += xrt-mgmt.o > + > +xrt-mgmt-objs := \ > + xmgmt-drv.o \ > + dt-test.dtb.o > + > +ccflags-y := -I$(FULL_XRT_PATH)/include > diff --git a/drivers/fpga/xrt/mgmt/dt-test.dts b/drivers/fpga/xrt/mgmt/dt-test.dts > new file mode 100644 > index 000000000000..68dbcb7fd79d > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/dt-test.dts > @@ -0,0 +1,12 @@ > +// SPDX-License-Identifier: GPL-2.0 > +/dts-v1/; > + > +/ { > + compatible = "xlnx,alveo-partition", "simple-bus"; > + #address-cells = <2>; > + #size-cells = <2>; > + pr_isolate_ulp@0,41000 { > + compatible = "xlnx,alveo-pr-isolation"; > + reg = <0x0 0x41000 0x0 0x1000>; > + }; > +}; > diff --git a/drivers/fpga/xrt/mgmt/dt-test.h b/drivers/fpga/xrt/mgmt/dt-test.h > new file mode 100644 > index 000000000000..6ec4203afbd2 > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/dt-test.h > @@ -0,0 +1,15 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +/* > + * Copyright (C) 2020-2022 Xilinx, Inc. > + * > + * Authors: > + * Lizhi Hou <lizhih@xilinx.com> > + */ > + > +#ifndef _DT_TEST_H_ > +#define _DT_TEST_H_ > + > +extern u8 __dtb_dt_test_begin[]; > +extern u8 __dtb_dt_test_end[]; these externs are also in lib-drv.h, this could be a duplicate file > + > +#endif /* _DT_TEST_H_ */ > diff --git a/drivers/fpga/xrt/mgmt/xmgmt-drv.c b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > new file mode 100644 > index 000000000000..87abe5b86e0b > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > @@ -0,0 +1,158 @@ > +// SPDX-License-Identifier: GPL-2.0 > +/* > + * Xilinx Alveo Management Function Driver > + * > + * Copyright (C) 2020-2022 Xilinx, Inc. > + * > + * Authors: > + * Cheng Zhen <maxz@xilinx.com> > + * Lizhi Hou <lizhih@xilinx.com> > + */ > + > +#include <linux/module.h> > +#include <linux/pci.h> > +#include <linux/aer.h> > +#include <linux/vmalloc.h> > +#include <linux/delay.h> > +#include "xpartition.h" > +#include "dt-test.h" > + > +#define XMGMT_MODULE_NAME "xrt-mgmt" > + > +#define XMGMT_PDEV(xm) ((xm)->pdev) > +#define XMGMT_DEV(xm) (&(XMGMT_PDEV(xm)->dev)) > +#define xmgmt_err(xm, fmt, args...) \ > + dev_err(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define xmgmt_warn(xm, fmt, args...) \ > + dev_warn(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define xmgmt_info(xm, fmt, args...) \ > + dev_info(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define xmgmt_dbg(xm, fmt, args...) \ > + dev_dbg(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define XMGMT_DEV_ID(_pcidev) \ > + ({ typeof(_pcidev) (pcidev) = (_pcidev); \ > + ((pci_domain_nr((pcidev)->bus) << 16) | \ > + PCI_DEVID((pcidev)->bus->number, (pcidev)->devfn)); }) > + > +#define XRT_MAX_READRQ 512 > + > +/* PCI Device IDs */ > +#define PCI_DEVICE_ID_U50 0x5020 > +static const struct pci_device_id xmgmt_pci_ids[] = { > + { PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_U50), }, /* Alveo U50 */ > + { 0, } > +}; > + > +struct xmgmt { > + struct pci_dev *pdev; > + void *base_partition; > + remove empty nl > + bool ready; > +}; > + > +static int xmgmt_config_pci(struct xmgmt *xm) > +{ > + struct pci_dev *pdev = XMGMT_PDEV(xm); > + int rc; > + > + rc = pcim_enable_device(pdev); > + if (rc < 0) { > + xmgmt_err(xm, "failed to enable device: %d", rc); > + return rc; > + } > + > + rc = pci_enable_pcie_error_reporting(pdev); > + if (rc) > + xmgmt_warn(xm, "failed to enable AER: %d", rc); > + > + pci_set_master(pdev); > + > + rc = pcie_get_readrq(pdev); > + if (rc > XRT_MAX_READRQ) > + pcie_set_readrq(pdev, XRT_MAX_READRQ); > + return 0; > +} > + > +static int xmgmt_probe(struct pci_dev *pdev, const struct pci_device_id *id) > +{ > + struct xrt_partition_range ranges[PCI_NUM_RESOURCES]; > + struct xrt_partition_info xp_info = { 0 }; > + struct device *dev = &pdev->dev; > + int ret, i, idx = 0; > + struct xmgmt *xm; > + > + xm = devm_kzalloc(dev, sizeof(*xm), GFP_KERNEL); > + if (!xm) > + return -ENOMEM; > + xm->pdev = pdev; > + pci_set_drvdata(pdev, xm); > + > + ret = xmgmt_config_pci(xm); > + if (ret) > + goto failed; > + > + for (i = 0; i < PCI_NUM_RESOURCES; i++) { > + if (pci_resource_len(pdev, i) > 0) { > + ranges[idx].bar_idx = i; > + ranges[idx].base = pci_resource_start(pdev, i); > + ranges[idx].size = pci_resource_len(pdev, i); > + idx++; > + } > + } > + xp_info.num_range = idx; > + xp_info.ranges = ranges; > + xp_info.fdt = __dtb_dt_test_begin; > + xp_info.fdt_len = (u32)(__dtb_dt_test_end - __dtb_dt_test_begin); > + ret = xrt_partition_create(&pdev->dev, &xp_info, &xm->base_partition); > + if (ret) > + goto failed; > + > + xmgmt_info(xm, "%s started successfully", XMGMT_MODULE_NAME); > + return 0; > + > +failed: > + if (xm->base_partition) This if-check and xrt_partition can be removed. xm->base_partition is only set when xrt_partition_create is successful > + xrt_partition_destroy(xm->base_partition); > + pci_set_drvdata(pdev, NULL); > + return ret; > +} > + > +static void xmgmt_remove(struct pci_dev *pdev) > +{ > + struct xmgmt *xm = pci_get_drvdata(pdev); > + > + xrt_partition_destroy(xm->base_partition); > + pci_disable_pcie_error_reporting(xm->pdev); > + xmgmt_info(xm, "%s cleaned up successfully", XMGMT_MODULE_NAME); > +} > + > +static struct pci_driver xmgmt_driver = { > + .name = XMGMT_MODULE_NAME, > + .id_table = xmgmt_pci_ids, > + .probe = xmgmt_probe, > + .remove = xmgmt_remove, > +}; > + > +static int __init xmgmt_init(void) > +{ > + int res = 0; This is a dead assignment, keep if you want. Tom > + > + res = pci_register_driver(&xmgmt_driver); > + if (res) > + return res; > + > + return 0; > +} > + > +static __exit void xmgmt_exit(void) > +{ > + pci_unregister_driver(&xmgmt_driver); > +} > + > +module_init(xmgmt_init); > +module_exit(xmgmt_exit); > + > +MODULE_DEVICE_TABLE(pci, xmgmt_pci_ids); > +MODULE_AUTHOR("XRT Team <runtime@xilinx.com>"); > +MODULE_DESCRIPTION("Xilinx Alveo management function driver"); > +MODULE_LICENSE("GPL v2");
On Wed, Jan 05, 2022 at 02:50:13PM -0800, Lizhi Hou wrote: > The PCIE device driver which attaches to management function on Alveo > devices. It instantiates one or more partition. Each partition consists > a set of hardward endpoints. A flat device tree is associated with each > partition. The first version of this driver uses test version flat device > tree and call xrt lib API to unflatten it. > > Signed-off-by: Sonal Santan <sonal.santan@xilinx.com> > Signed-off-by: Max Zhen <max.zhen@xilinx.com> > Signed-off-by: Lizhi Hou <lizhi.hou@xilinx.com> > --- > drivers/fpga/Makefile | 1 + > drivers/fpga/xrt/Kconfig | 1 + > drivers/fpga/xrt/mgmt/Kconfig | 14 +++ > drivers/fpga/xrt/mgmt/Makefile | 16 +++ > drivers/fpga/xrt/mgmt/dt-test.dts | 12 +++ > drivers/fpga/xrt/mgmt/dt-test.h | 15 +++ > drivers/fpga/xrt/mgmt/xmgmt-drv.c | 158 ++++++++++++++++++++++++++++++ > 7 files changed, 217 insertions(+) > create mode 100644 drivers/fpga/xrt/mgmt/Kconfig > create mode 100644 drivers/fpga/xrt/mgmt/Makefile > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.dts > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.h > create mode 100644 drivers/fpga/xrt/mgmt/xmgmt-drv.c > > diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile > index 5bd41cf4c7ec..544e2144878f 100644 > --- a/drivers/fpga/Makefile > +++ b/drivers/fpga/Makefile > @@ -52,3 +52,4 @@ obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o > > # XRT drivers for Alveo > obj-$(CONFIG_FPGA_XRT_LIB) += xrt/lib/ > +obj-$(CONFIG_FPGA_XRT_XMGMT) += xrt/mgmt/ > diff --git a/drivers/fpga/xrt/Kconfig b/drivers/fpga/xrt/Kconfig > index 04c3bb5aaf4f..50422f77c6df 100644 > --- a/drivers/fpga/xrt/Kconfig > +++ b/drivers/fpga/xrt/Kconfig > @@ -4,3 +4,4 @@ > # > > source "drivers/fpga/xrt/lib/Kconfig" > +source "drivers/fpga/xrt/mgmt/Kconfig" > diff --git a/drivers/fpga/xrt/mgmt/Kconfig b/drivers/fpga/xrt/mgmt/Kconfig > new file mode 100644 > index 000000000000..a978747482be > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/Kconfig > @@ -0,0 +1,14 @@ > +# SPDX-License-Identifier: GPL-2.0-only > +# > +# Xilinx XRT FPGA device configuration > +# > + > +config FPGA_XRT_XMGMT > + tristate "Xilinx Alveo Management Driver" > + depends on FPGA_XRT_LIB > + select FPGA_BRIDGE > + select FPGA_REGION > + help > + Select this option to enable XRT PCIe driver for Xilinx Alveo FPGA. > + This driver provides interfaces for userspace application to access > + Alveo FPGA device. > diff --git a/drivers/fpga/xrt/mgmt/Makefile b/drivers/fpga/xrt/mgmt/Makefile > new file mode 100644 > index 000000000000..c5134bf71cca > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/Makefile > @@ -0,0 +1,16 @@ > +# SPDX-License-Identifier: GPL-2.0 > +# > +# Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved. > +# > +# Authors: Sonal.Santan@xilinx.com > +# > + > +FULL_XRT_PATH=$(srctree)/$(src)/.. > + > +obj-$(CONFIG_FPGA_XRT_LIB) += xrt-mgmt.o > + > +xrt-mgmt-objs := \ > + xmgmt-drv.o \ > + dt-test.dtb.o > + > +ccflags-y := -I$(FULL_XRT_PATH)/include > diff --git a/drivers/fpga/xrt/mgmt/dt-test.dts b/drivers/fpga/xrt/mgmt/dt-test.dts > new file mode 100644 > index 000000000000..68dbcb7fd79d > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/dt-test.dts > @@ -0,0 +1,12 @@ > +// SPDX-License-Identifier: GPL-2.0 > +/dts-v1/; > + > +/ { > + compatible = "xlnx,alveo-partition", "simple-bus"; > + #address-cells = <2>; > + #size-cells = <2>; > + pr_isolate_ulp@0,41000 { > + compatible = "xlnx,alveo-pr-isolation"; > + reg = <0x0 0x41000 0x0 0x1000>; > + }; > +}; I remember Rob's comments: "we'd need to create a base tree (if there isn't one) with nodes for the USB or PCI device(s) and then an overlay for the device can be applied to those nodes." https://lore.kernel.org/linux-fpga/CAL_JsqJfyRymB=VxLuQqLpep+Q1Eie48dobv9sC5OizDz0d2DQ@mail.gmail.com/ So could we firstly create a pci device node under the of_root when the driver probing, then add the partition DTs under the pci device node by overlay machenism. I'm considering if finally we could leverage the existing of-fpga-region that reprograms and enumerates FPGA sub devices by overlay. Open for discussion. Thanks, Yilun > diff --git a/drivers/fpga/xrt/mgmt/dt-test.h b/drivers/fpga/xrt/mgmt/dt-test.h > new file mode 100644 > index 000000000000..6ec4203afbd2 > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/dt-test.h > @@ -0,0 +1,15 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +/* > + * Copyright (C) 2020-2022 Xilinx, Inc. > + * > + * Authors: > + * Lizhi Hou <lizhih@xilinx.com> > + */ > + > +#ifndef _DT_TEST_H_ > +#define _DT_TEST_H_ > + > +extern u8 __dtb_dt_test_begin[]; > +extern u8 __dtb_dt_test_end[]; > + > +#endif /* _DT_TEST_H_ */ > diff --git a/drivers/fpga/xrt/mgmt/xmgmt-drv.c b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > new file mode 100644 > index 000000000000..87abe5b86e0b > --- /dev/null > +++ b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > @@ -0,0 +1,158 @@ > +// SPDX-License-Identifier: GPL-2.0 > +/* > + * Xilinx Alveo Management Function Driver > + * > + * Copyright (C) 2020-2022 Xilinx, Inc. > + * > + * Authors: > + * Cheng Zhen <maxz@xilinx.com> > + * Lizhi Hou <lizhih@xilinx.com> > + */ > + > +#include <linux/module.h> > +#include <linux/pci.h> > +#include <linux/aer.h> > +#include <linux/vmalloc.h> > +#include <linux/delay.h> > +#include "xpartition.h" > +#include "dt-test.h" > + > +#define XMGMT_MODULE_NAME "xrt-mgmt" > + > +#define XMGMT_PDEV(xm) ((xm)->pdev) > +#define XMGMT_DEV(xm) (&(XMGMT_PDEV(xm)->dev)) > +#define xmgmt_err(xm, fmt, args...) \ > + dev_err(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define xmgmt_warn(xm, fmt, args...) \ > + dev_warn(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define xmgmt_info(xm, fmt, args...) \ > + dev_info(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define xmgmt_dbg(xm, fmt, args...) \ > + dev_dbg(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > +#define XMGMT_DEV_ID(_pcidev) \ > + ({ typeof(_pcidev) (pcidev) = (_pcidev); \ > + ((pci_domain_nr((pcidev)->bus) << 16) | \ > + PCI_DEVID((pcidev)->bus->number, (pcidev)->devfn)); }) > + > +#define XRT_MAX_READRQ 512 > + > +/* PCI Device IDs */ > +#define PCI_DEVICE_ID_U50 0x5020 > +static const struct pci_device_id xmgmt_pci_ids[] = { > + { PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_U50), }, /* Alveo U50 */ > + { 0, } > +}; > + > +struct xmgmt { > + struct pci_dev *pdev; > + void *base_partition; > + > + bool ready; > +}; > + > +static int xmgmt_config_pci(struct xmgmt *xm) > +{ > + struct pci_dev *pdev = XMGMT_PDEV(xm); > + int rc; > + > + rc = pcim_enable_device(pdev); > + if (rc < 0) { > + xmgmt_err(xm, "failed to enable device: %d", rc); > + return rc; > + } > + > + rc = pci_enable_pcie_error_reporting(pdev); > + if (rc) > + xmgmt_warn(xm, "failed to enable AER: %d", rc); > + > + pci_set_master(pdev); > + > + rc = pcie_get_readrq(pdev); > + if (rc > XRT_MAX_READRQ) > + pcie_set_readrq(pdev, XRT_MAX_READRQ); > + return 0; > +} > + > +static int xmgmt_probe(struct pci_dev *pdev, const struct pci_device_id *id) > +{ > + struct xrt_partition_range ranges[PCI_NUM_RESOURCES]; > + struct xrt_partition_info xp_info = { 0 }; > + struct device *dev = &pdev->dev; > + int ret, i, idx = 0; > + struct xmgmt *xm; > + > + xm = devm_kzalloc(dev, sizeof(*xm), GFP_KERNEL); > + if (!xm) > + return -ENOMEM; > + xm->pdev = pdev; > + pci_set_drvdata(pdev, xm); > + > + ret = xmgmt_config_pci(xm); > + if (ret) > + goto failed; > + > + for (i = 0; i < PCI_NUM_RESOURCES; i++) { > + if (pci_resource_len(pdev, i) > 0) { > + ranges[idx].bar_idx = i; > + ranges[idx].base = pci_resource_start(pdev, i); > + ranges[idx].size = pci_resource_len(pdev, i); > + idx++; > + } > + } > + xp_info.num_range = idx; > + xp_info.ranges = ranges; > + xp_info.fdt = __dtb_dt_test_begin; > + xp_info.fdt_len = (u32)(__dtb_dt_test_end - __dtb_dt_test_begin); > + ret = xrt_partition_create(&pdev->dev, &xp_info, &xm->base_partition); > + if (ret) > + goto failed; > + > + xmgmt_info(xm, "%s started successfully", XMGMT_MODULE_NAME); > + return 0; > + > +failed: > + if (xm->base_partition) > + xrt_partition_destroy(xm->base_partition); > + pci_set_drvdata(pdev, NULL); > + return ret; > +} > + > +static void xmgmt_remove(struct pci_dev *pdev) > +{ > + struct xmgmt *xm = pci_get_drvdata(pdev); > + > + xrt_partition_destroy(xm->base_partition); > + pci_disable_pcie_error_reporting(xm->pdev); > + xmgmt_info(xm, "%s cleaned up successfully", XMGMT_MODULE_NAME); > +} > + > +static struct pci_driver xmgmt_driver = { > + .name = XMGMT_MODULE_NAME, > + .id_table = xmgmt_pci_ids, > + .probe = xmgmt_probe, > + .remove = xmgmt_remove, > +}; > + > +static int __init xmgmt_init(void) > +{ > + int res = 0; > + > + res = pci_register_driver(&xmgmt_driver); > + if (res) > + return res; > + > + return 0; > +} > + > +static __exit void xmgmt_exit(void) > +{ > + pci_unregister_driver(&xmgmt_driver); > +} > + > +module_init(xmgmt_init); > +module_exit(xmgmt_exit); > + > +MODULE_DEVICE_TABLE(pci, xmgmt_pci_ids); > +MODULE_AUTHOR("XRT Team <runtime@xilinx.com>"); > +MODULE_DESCRIPTION("Xilinx Alveo management function driver"); > +MODULE_LICENSE("GPL v2"); > -- > 2.27.0
On 1/10/22 11:00 PM, Xu Yilun wrote: > = > > On Wed, Jan 05, 2022 at 02:50:13PM -0800, Lizhi Hou wrote: >> The PCIE device driver which attaches to management function on Alveo >> devices. It instantiates one or more partition. Each partition consists >> a set of hardward endpoints. A flat device tree is associated with each >> partition. The first version of this driver uses test version flat device >> tree and call xrt lib API to unflatten it. >> >> Signed-off-by: Sonal Santan <sonal.santan@xilinx.com> >> Signed-off-by: Max Zhen <max.zhen@xilinx.com> >> Signed-off-by: Lizhi Hou <lizhi.hou@xilinx.com> >> --- >> drivers/fpga/Makefile | 1 + >> drivers/fpga/xrt/Kconfig | 1 + >> drivers/fpga/xrt/mgmt/Kconfig | 14 +++ >> drivers/fpga/xrt/mgmt/Makefile | 16 +++ >> drivers/fpga/xrt/mgmt/dt-test.dts | 12 +++ >> drivers/fpga/xrt/mgmt/dt-test.h | 15 +++ >> drivers/fpga/xrt/mgmt/xmgmt-drv.c | 158 ++++++++++++++++++++++++++++++ >> 7 files changed, 217 insertions(+) >> create mode 100644 drivers/fpga/xrt/mgmt/Kconfig >> create mode 100644 drivers/fpga/xrt/mgmt/Makefile >> create mode 100644 drivers/fpga/xrt/mgmt/dt-test.dts >> create mode 100644 drivers/fpga/xrt/mgmt/dt-test.h >> create mode 100644 drivers/fpga/xrt/mgmt/xmgmt-drv.c >> >> diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile >> index 5bd41cf4c7ec..544e2144878f 100644 >> --- a/drivers/fpga/Makefile >> +++ b/drivers/fpga/Makefile >> @@ -52,3 +52,4 @@ obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o >> >> # XRT drivers for Alveo >> obj-$(CONFIG_FPGA_XRT_LIB) += xrt/lib/ >> +obj-$(CONFIG_FPGA_XRT_XMGMT) += xrt/mgmt/ >> diff --git a/drivers/fpga/xrt/Kconfig b/drivers/fpga/xrt/Kconfig >> index 04c3bb5aaf4f..50422f77c6df 100644 >> --- a/drivers/fpga/xrt/Kconfig >> +++ b/drivers/fpga/xrt/Kconfig >> @@ -4,3 +4,4 @@ >> # >> >> source "drivers/fpga/xrt/lib/Kconfig" >> +source "drivers/fpga/xrt/mgmt/Kconfig" >> diff --git a/drivers/fpga/xrt/mgmt/Kconfig b/drivers/fpga/xrt/mgmt/Kconfig >> new file mode 100644 >> index 000000000000..a978747482be >> --- /dev/null >> +++ b/drivers/fpga/xrt/mgmt/Kconfig >> @@ -0,0 +1,14 @@ >> +# SPDX-License-Identifier: GPL-2.0-only >> +# >> +# Xilinx XRT FPGA device configuration >> +# >> + >> +config FPGA_XRT_XMGMT >> + tristate "Xilinx Alveo Management Driver" >> + depends on FPGA_XRT_LIB >> + select FPGA_BRIDGE >> + select FPGA_REGION >> + help >> + Select this option to enable XRT PCIe driver for Xilinx Alveo FPGA. >> + This driver provides interfaces for userspace application to access >> + Alveo FPGA device. >> diff --git a/drivers/fpga/xrt/mgmt/Makefile b/drivers/fpga/xrt/mgmt/Makefile >> new file mode 100644 >> index 000000000000..c5134bf71cca >> --- /dev/null >> +++ b/drivers/fpga/xrt/mgmt/Makefile >> @@ -0,0 +1,16 @@ >> +# SPDX-License-Identifier: GPL-2.0 >> +# >> +# Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved. >> +# >> +# Authors: Sonal.Santan@xilinx.com >> +# >> + >> +FULL_XRT_PATH=$(srctree)/$(src)/.. >> + >> +obj-$(CONFIG_FPGA_XRT_LIB) += xrt-mgmt.o >> + >> +xrt-mgmt-objs := \ >> + xmgmt-drv.o \ >> + dt-test.dtb.o >> + >> +ccflags-y := -I$(FULL_XRT_PATH)/include >> diff --git a/drivers/fpga/xrt/mgmt/dt-test.dts b/drivers/fpga/xrt/mgmt/dt-test.dts >> new file mode 100644 >> index 000000000000..68dbcb7fd79d >> --- /dev/null >> +++ b/drivers/fpga/xrt/mgmt/dt-test.dts >> @@ -0,0 +1,12 @@ >> +// SPDX-License-Identifier: GPL-2.0 >> +/dts-v1/; >> + >> +/ { >> + compatible = "xlnx,alveo-partition", "simple-bus"; >> + #address-cells = <2>; >> + #size-cells = <2>; >> + pr_isolate_ulp@0,41000 { >> + compatible = "xlnx,alveo-pr-isolation"; >> + reg = <0x0 0x41000 0x0 0x1000>; >> + }; >> +}; > I remember Rob's comments: > > "we'd need to create a base tree (if there isn't one) with nodes > for the USB or PCI device(s) and then an overlay for the device can be > applied to those nodes." > > https://lore.kernel.org/linux-fpga/CAL_JsqJfyRymB=VxLuQqLpep+Q1Eie48dobv9sC5OizDz0d2DQ@mail.gmail.com/ > > So could we firstly create a pci device node under the of_root when > the driver probing, then add the partition DTs under the pci device node > by overlay machenism. > > I'm considering if finally we could leverage the existing of-fpga-region > that reprograms and enumerates FPGA sub devices by overlay. > > Open for discussion. I would list the main ideas of our implementation 1. “alveo-partition (simple-bus)” node is created under of_root in xmgmt probe function. “simple-bus” is used as it fits well and we can modify it to “fpga-region” when we refactor fpga-mgr/region/bridge part. 2. we are the first pcie device using flatten device tree to describe sub devices. Adding device tree nodes for all pcie device sounds intrusive. We can expend to all pcie device when we have another user of this feature. 3. of_overlay_fdt_apply() does not support overlaying to a dynamic generated parent node. Thus, we use of_dynamic functions of_changeset_attach_node() instead. (similar to drivers/pci/hotplug/pnv_php.c,) . If needed ,we can consider to add parameter “parent node” to of_overlay_fdt_apply() to support dynamical overlay case. Thanks, Lizhi > > Thanks, > Yilun > >> diff --git a/drivers/fpga/xrt/mgmt/dt-test.h b/drivers/fpga/xrt/mgmt/dt-test.h >> new file mode 100644 >> index 000000000000..6ec4203afbd2 >> --- /dev/null >> +++ b/drivers/fpga/xrt/mgmt/dt-test.h >> @@ -0,0 +1,15 @@ >> +/* SPDX-License-Identifier: GPL-2.0 */ >> +/* >> + * Copyright (C) 2020-2022 Xilinx, Inc. >> + * >> + * Authors: >> + * Lizhi Hou <lizhih@xilinx.com> >> + */ >> + >> +#ifndef _DT_TEST_H_ >> +#define _DT_TEST_H_ >> + >> +extern u8 __dtb_dt_test_begin[]; >> +extern u8 __dtb_dt_test_end[]; >> + >> +#endif /* _DT_TEST_H_ */ >> diff --git a/drivers/fpga/xrt/mgmt/xmgmt-drv.c b/drivers/fpga/xrt/mgmt/xmgmt-drv.c >> new file mode 100644 >> index 000000000000..87abe5b86e0b >> --- /dev/null >> +++ b/drivers/fpga/xrt/mgmt/xmgmt-drv.c >> @@ -0,0 +1,158 @@ >> +// SPDX-License-Identifier: GPL-2.0 >> +/* >> + * Xilinx Alveo Management Function Driver >> + * >> + * Copyright (C) 2020-2022 Xilinx, Inc. >> + * >> + * Authors: >> + * Cheng Zhen <maxz@xilinx.com> >> + * Lizhi Hou <lizhih@xilinx.com> >> + */ >> + >> +#include <linux/module.h> >> +#include <linux/pci.h> >> +#include <linux/aer.h> >> +#include <linux/vmalloc.h> >> +#include <linux/delay.h> >> +#include "xpartition.h" >> +#include "dt-test.h" >> + >> +#define XMGMT_MODULE_NAME "xrt-mgmt" >> + >> +#define XMGMT_PDEV(xm) ((xm)->pdev) >> +#define XMGMT_DEV(xm) (&(XMGMT_PDEV(xm)->dev)) >> +#define xmgmt_err(xm, fmt, args...) \ >> + dev_err(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >> +#define xmgmt_warn(xm, fmt, args...) \ >> + dev_warn(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >> +#define xmgmt_info(xm, fmt, args...) \ >> + dev_info(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >> +#define xmgmt_dbg(xm, fmt, args...) \ >> + dev_dbg(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >> +#define XMGMT_DEV_ID(_pcidev) \ >> + ({ typeof(_pcidev) (pcidev) = (_pcidev); \ >> + ((pci_domain_nr((pcidev)->bus) << 16) | \ >> + PCI_DEVID((pcidev)->bus->number, (pcidev)->devfn)); }) >> + >> +#define XRT_MAX_READRQ 512 >> + >> +/* PCI Device IDs */ >> +#define PCI_DEVICE_ID_U50 0x5020 >> +static const struct pci_device_id xmgmt_pci_ids[] = { >> + { PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_U50), }, /* Alveo U50 */ >> + { 0, } >> +}; >> + >> +struct xmgmt { >> + struct pci_dev *pdev; >> + void *base_partition; >> + >> + bool ready; >> +}; >> + >> +static int xmgmt_config_pci(struct xmgmt *xm) >> +{ >> + struct pci_dev *pdev = XMGMT_PDEV(xm); >> + int rc; >> + >> + rc = pcim_enable_device(pdev); >> + if (rc < 0) { >> + xmgmt_err(xm, "failed to enable device: %d", rc); >> + return rc; >> + } >> + >> + rc = pci_enable_pcie_error_reporting(pdev); >> + if (rc) >> + xmgmt_warn(xm, "failed to enable AER: %d", rc); >> + >> + pci_set_master(pdev); >> + >> + rc = pcie_get_readrq(pdev); >> + if (rc > XRT_MAX_READRQ) >> + pcie_set_readrq(pdev, XRT_MAX_READRQ); >> + return 0; >> +} >> + >> +static int xmgmt_probe(struct pci_dev *pdev, const struct pci_device_id *id) >> +{ >> + struct xrt_partition_range ranges[PCI_NUM_RESOURCES]; >> + struct xrt_partition_info xp_info = { 0 }; >> + struct device *dev = &pdev->dev; >> + int ret, i, idx = 0; >> + struct xmgmt *xm; >> + >> + xm = devm_kzalloc(dev, sizeof(*xm), GFP_KERNEL); >> + if (!xm) >> + return -ENOMEM; >> + xm->pdev = pdev; >> + pci_set_drvdata(pdev, xm); >> + >> + ret = xmgmt_config_pci(xm); >> + if (ret) >> + goto failed; >> + >> + for (i = 0; i < PCI_NUM_RESOURCES; i++) { >> + if (pci_resource_len(pdev, i) > 0) { >> + ranges[idx].bar_idx = i; >> + ranges[idx].base = pci_resource_start(pdev, i); >> + ranges[idx].size = pci_resource_len(pdev, i); >> + idx++; >> + } >> + } >> + xp_info.num_range = idx; >> + xp_info.ranges = ranges; >> + xp_info.fdt = __dtb_dt_test_begin; >> + xp_info.fdt_len = (u32)(__dtb_dt_test_end - __dtb_dt_test_begin); >> + ret = xrt_partition_create(&pdev->dev, &xp_info, &xm->base_partition); >> + if (ret) >> + goto failed; >> + >> + xmgmt_info(xm, "%s started successfully", XMGMT_MODULE_NAME); >> + return 0; >> + >> +failed: >> + if (xm->base_partition) >> + xrt_partition_destroy(xm->base_partition); >> + pci_set_drvdata(pdev, NULL); >> + return ret; >> +} >> + >> +static void xmgmt_remove(struct pci_dev *pdev) >> +{ >> + struct xmgmt *xm = pci_get_drvdata(pdev); >> + >> + xrt_partition_destroy(xm->base_partition); >> + pci_disable_pcie_error_reporting(xm->pdev); >> + xmgmt_info(xm, "%s cleaned up successfully", XMGMT_MODULE_NAME); >> +} >> + >> +static struct pci_driver xmgmt_driver = { >> + .name = XMGMT_MODULE_NAME, >> + .id_table = xmgmt_pci_ids, >> + .probe = xmgmt_probe, >> + .remove = xmgmt_remove, >> +}; >> + >> +static int __init xmgmt_init(void) >> +{ >> + int res = 0; >> + >> + res = pci_register_driver(&xmgmt_driver); >> + if (res) >> + return res; >> + >> + return 0; >> +} >> + >> +static __exit void xmgmt_exit(void) >> +{ >> + pci_unregister_driver(&xmgmt_driver); >> +} >> + >> +module_init(xmgmt_init); >> +module_exit(xmgmt_exit); >> + >> +MODULE_DEVICE_TABLE(pci, xmgmt_pci_ids); >> +MODULE_AUTHOR("XRT Team <runtime@xilinx.com>"); >> +MODULE_DESCRIPTION("Xilinx Alveo management function driver"); >> +MODULE_LICENSE("GPL v2"); >> -- >> 2.27.0
On Thu, Jan 13, 2022 at 03:41:47PM -0800, Lizhi Hou wrote: > > On 1/10/22 11:00 PM, Xu Yilun wrote: > > = > > > > On Wed, Jan 05, 2022 at 02:50:13PM -0800, Lizhi Hou wrote: > > > The PCIE device driver which attaches to management function on Alveo > > > devices. It instantiates one or more partition. Each partition consists > > > a set of hardward endpoints. A flat device tree is associated with each > > > partition. The first version of this driver uses test version flat device > > > tree and call xrt lib API to unflatten it. > > > > > > Signed-off-by: Sonal Santan <sonal.santan@xilinx.com> > > > Signed-off-by: Max Zhen <max.zhen@xilinx.com> > > > Signed-off-by: Lizhi Hou <lizhi.hou@xilinx.com> > > > --- > > > drivers/fpga/Makefile | 1 + > > > drivers/fpga/xrt/Kconfig | 1 + > > > drivers/fpga/xrt/mgmt/Kconfig | 14 +++ > > > drivers/fpga/xrt/mgmt/Makefile | 16 +++ > > > drivers/fpga/xrt/mgmt/dt-test.dts | 12 +++ > > > drivers/fpga/xrt/mgmt/dt-test.h | 15 +++ > > > drivers/fpga/xrt/mgmt/xmgmt-drv.c | 158 ++++++++++++++++++++++++++++++ > > > 7 files changed, 217 insertions(+) > > > create mode 100644 drivers/fpga/xrt/mgmt/Kconfig > > > create mode 100644 drivers/fpga/xrt/mgmt/Makefile > > > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.dts > > > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.h > > > create mode 100644 drivers/fpga/xrt/mgmt/xmgmt-drv.c > > > > > > diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile > > > index 5bd41cf4c7ec..544e2144878f 100644 > > > --- a/drivers/fpga/Makefile > > > +++ b/drivers/fpga/Makefile > > > @@ -52,3 +52,4 @@ obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o > > > > > > # XRT drivers for Alveo > > > obj-$(CONFIG_FPGA_XRT_LIB) += xrt/lib/ > > > +obj-$(CONFIG_FPGA_XRT_XMGMT) += xrt/mgmt/ > > > diff --git a/drivers/fpga/xrt/Kconfig b/drivers/fpga/xrt/Kconfig > > > index 04c3bb5aaf4f..50422f77c6df 100644 > > > --- a/drivers/fpga/xrt/Kconfig > > > +++ b/drivers/fpga/xrt/Kconfig > > > @@ -4,3 +4,4 @@ > > > # > > > > > > source "drivers/fpga/xrt/lib/Kconfig" > > > +source "drivers/fpga/xrt/mgmt/Kconfig" > > > diff --git a/drivers/fpga/xrt/mgmt/Kconfig b/drivers/fpga/xrt/mgmt/Kconfig > > > new file mode 100644 > > > index 000000000000..a978747482be > > > --- /dev/null > > > +++ b/drivers/fpga/xrt/mgmt/Kconfig > > > @@ -0,0 +1,14 @@ > > > +# SPDX-License-Identifier: GPL-2.0-only > > > +# > > > +# Xilinx XRT FPGA device configuration > > > +# > > > + > > > +config FPGA_XRT_XMGMT > > > + tristate "Xilinx Alveo Management Driver" > > > + depends on FPGA_XRT_LIB > > > + select FPGA_BRIDGE > > > + select FPGA_REGION > > > + help > > > + Select this option to enable XRT PCIe driver for Xilinx Alveo FPGA. > > > + This driver provides interfaces for userspace application to access > > > + Alveo FPGA device. > > > diff --git a/drivers/fpga/xrt/mgmt/Makefile b/drivers/fpga/xrt/mgmt/Makefile > > > new file mode 100644 > > > index 000000000000..c5134bf71cca > > > --- /dev/null > > > +++ b/drivers/fpga/xrt/mgmt/Makefile > > > @@ -0,0 +1,16 @@ > > > +# SPDX-License-Identifier: GPL-2.0 > > > +# > > > +# Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved. > > > +# > > > +# Authors: Sonal.Santan@xilinx.com > > > +# > > > + > > > +FULL_XRT_PATH=$(srctree)/$(src)/.. > > > + > > > +obj-$(CONFIG_FPGA_XRT_LIB) += xrt-mgmt.o > > > + > > > +xrt-mgmt-objs := \ > > > + xmgmt-drv.o \ > > > + dt-test.dtb.o > > > + > > > +ccflags-y := -I$(FULL_XRT_PATH)/include > > > diff --git a/drivers/fpga/xrt/mgmt/dt-test.dts b/drivers/fpga/xrt/mgmt/dt-test.dts > > > new file mode 100644 > > > index 000000000000..68dbcb7fd79d > > > --- /dev/null > > > +++ b/drivers/fpga/xrt/mgmt/dt-test.dts > > > @@ -0,0 +1,12 @@ > > > +// SPDX-License-Identifier: GPL-2.0 > > > +/dts-v1/; > > > + > > > +/ { > > > + compatible = "xlnx,alveo-partition", "simple-bus"; > > > + #address-cells = <2>; > > > + #size-cells = <2>; > > > + pr_isolate_ulp@0,41000 { > > > + compatible = "xlnx,alveo-pr-isolation"; > > > + reg = <0x0 0x41000 0x0 0x1000>; > > > + }; > > > +}; > > I remember Rob's comments: > > > > "we'd need to create a base tree (if there isn't one) with nodes > > for the USB or PCI device(s) and then an overlay for the device can be > > applied to those nodes." > > > > https://lore.kernel.org/linux-fpga/CAL_JsqJfyRymB=VxLuQqLpep+Q1Eie48dobv9sC5OizDz0d2DQ@mail.gmail.com/ > > > > So could we firstly create a pci device node under the of_root when > > the driver probing, then add the partition DTs under the pci device node > > by overlay machenism. > > > > I'm considering if finally we could leverage the existing of-fpga-region > > that reprograms and enumerates FPGA sub devices by overlay. > > > > Open for discussion. > > I would list the main ideas of our implementation > > 1. “alveo-partition (simple-bus)” node is created under of_root in xmgmt > probe function. “simple-bus” is used as it fits well and we can modify it to > “fpga-region” when we refactor fpga-mgr/region/bridge part. OK. "fpga-region" for an independent reprogramable region. And "simple-bus" for static board components. > 2. we are the first pcie device using flatten device tree to describe sub > devices. Adding device tree nodes for all pcie device sounds intrusive. We > can expend to all pcie device when we have another user of this feature. I don't mean we create DT nodes for all pcie devices in system, it is the device driver's choice to create the node for DT style enumeration of its sub devices. This is mainly for pcie based fpga cards. Now there exsits plenty of these cards in the world, and the enumeration of their sub devices could be a common concern. I think it's good we have a clear design at the beginning. > > 3. of_overlay_fdt_apply() does not support overlaying to a dynamic generated Yes that's the problem we have. > parent node. Thus, we use of_dynamic functions of_changeset_attach_node() > instead. (similar to drivers/pci/hotplug/pnv_php.c,) . If needed ,we can > consider to add parameter “parent node” to of_overlay_fdt_apply() to support > dynamical overlay case. I agree we do some investigation for this. Thanks, Yilun > > > Thanks, > > Lizhi > > > > > Thanks, > > Yilun > > > > > diff --git a/drivers/fpga/xrt/mgmt/dt-test.h b/drivers/fpga/xrt/mgmt/dt-test.h > > > new file mode 100644 > > > index 000000000000..6ec4203afbd2 > > > --- /dev/null > > > +++ b/drivers/fpga/xrt/mgmt/dt-test.h > > > @@ -0,0 +1,15 @@ > > > +/* SPDX-License-Identifier: GPL-2.0 */ > > > +/* > > > + * Copyright (C) 2020-2022 Xilinx, Inc. > > > + * > > > + * Authors: > > > + * Lizhi Hou <lizhih@xilinx.com> > > > + */ > > > + > > > +#ifndef _DT_TEST_H_ > > > +#define _DT_TEST_H_ > > > + > > > +extern u8 __dtb_dt_test_begin[]; > > > +extern u8 __dtb_dt_test_end[]; > > > + > > > +#endif /* _DT_TEST_H_ */ > > > diff --git a/drivers/fpga/xrt/mgmt/xmgmt-drv.c b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > > > new file mode 100644 > > > index 000000000000..87abe5b86e0b > > > --- /dev/null > > > +++ b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > > > @@ -0,0 +1,158 @@ > > > +// SPDX-License-Identifier: GPL-2.0 > > > +/* > > > + * Xilinx Alveo Management Function Driver > > > + * > > > + * Copyright (C) 2020-2022 Xilinx, Inc. > > > + * > > > + * Authors: > > > + * Cheng Zhen <maxz@xilinx.com> > > > + * Lizhi Hou <lizhih@xilinx.com> > > > + */ > > > + > > > +#include <linux/module.h> > > > +#include <linux/pci.h> > > > +#include <linux/aer.h> > > > +#include <linux/vmalloc.h> > > > +#include <linux/delay.h> > > > +#include "xpartition.h" > > > +#include "dt-test.h" > > > + > > > +#define XMGMT_MODULE_NAME "xrt-mgmt" > > > + > > > +#define XMGMT_PDEV(xm) ((xm)->pdev) > > > +#define XMGMT_DEV(xm) (&(XMGMT_PDEV(xm)->dev)) > > > +#define xmgmt_err(xm, fmt, args...) \ > > > + dev_err(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > +#define xmgmt_warn(xm, fmt, args...) \ > > > + dev_warn(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > +#define xmgmt_info(xm, fmt, args...) \ > > > + dev_info(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > +#define xmgmt_dbg(xm, fmt, args...) \ > > > + dev_dbg(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > +#define XMGMT_DEV_ID(_pcidev) \ > > > + ({ typeof(_pcidev) (pcidev) = (_pcidev); \ > > > + ((pci_domain_nr((pcidev)->bus) << 16) | \ > > > + PCI_DEVID((pcidev)->bus->number, (pcidev)->devfn)); }) > > > + > > > +#define XRT_MAX_READRQ 512 > > > + > > > +/* PCI Device IDs */ > > > +#define PCI_DEVICE_ID_U50 0x5020 > > > +static const struct pci_device_id xmgmt_pci_ids[] = { > > > + { PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_U50), }, /* Alveo U50 */ > > > + { 0, } > > > +}; > > > + > > > +struct xmgmt { > > > + struct pci_dev *pdev; > > > + void *base_partition; > > > + > > > + bool ready; > > > +}; > > > + > > > +static int xmgmt_config_pci(struct xmgmt *xm) > > > +{ > > > + struct pci_dev *pdev = XMGMT_PDEV(xm); > > > + int rc; > > > + > > > + rc = pcim_enable_device(pdev); > > > + if (rc < 0) { > > > + xmgmt_err(xm, "failed to enable device: %d", rc); > > > + return rc; > > > + } > > > + > > > + rc = pci_enable_pcie_error_reporting(pdev); > > > + if (rc) > > > + xmgmt_warn(xm, "failed to enable AER: %d", rc); > > > + > > > + pci_set_master(pdev); > > > + > > > + rc = pcie_get_readrq(pdev); > > > + if (rc > XRT_MAX_READRQ) > > > + pcie_set_readrq(pdev, XRT_MAX_READRQ); > > > + return 0; > > > +} > > > + > > > +static int xmgmt_probe(struct pci_dev *pdev, const struct pci_device_id *id) > > > +{ > > > + struct xrt_partition_range ranges[PCI_NUM_RESOURCES]; > > > + struct xrt_partition_info xp_info = { 0 }; > > > + struct device *dev = &pdev->dev; > > > + int ret, i, idx = 0; > > > + struct xmgmt *xm; > > > + > > > + xm = devm_kzalloc(dev, sizeof(*xm), GFP_KERNEL); > > > + if (!xm) > > > + return -ENOMEM; > > > + xm->pdev = pdev; > > > + pci_set_drvdata(pdev, xm); > > > + > > > + ret = xmgmt_config_pci(xm); > > > + if (ret) > > > + goto failed; > > > + > > > + for (i = 0; i < PCI_NUM_RESOURCES; i++) { > > > + if (pci_resource_len(pdev, i) > 0) { > > > + ranges[idx].bar_idx = i; > > > + ranges[idx].base = pci_resource_start(pdev, i); > > > + ranges[idx].size = pci_resource_len(pdev, i); > > > + idx++; > > > + } > > > + } > > > + xp_info.num_range = idx; > > > + xp_info.ranges = ranges; > > > + xp_info.fdt = __dtb_dt_test_begin; > > > + xp_info.fdt_len = (u32)(__dtb_dt_test_end - __dtb_dt_test_begin); > > > + ret = xrt_partition_create(&pdev->dev, &xp_info, &xm->base_partition); > > > + if (ret) > > > + goto failed; > > > + > > > + xmgmt_info(xm, "%s started successfully", XMGMT_MODULE_NAME); > > > + return 0; > > > + > > > +failed: > > > + if (xm->base_partition) > > > + xrt_partition_destroy(xm->base_partition); > > > + pci_set_drvdata(pdev, NULL); > > > + return ret; > > > +} > > > + > > > +static void xmgmt_remove(struct pci_dev *pdev) > > > +{ > > > + struct xmgmt *xm = pci_get_drvdata(pdev); > > > + > > > + xrt_partition_destroy(xm->base_partition); > > > + pci_disable_pcie_error_reporting(xm->pdev); > > > + xmgmt_info(xm, "%s cleaned up successfully", XMGMT_MODULE_NAME); > > > +} > > > + > > > +static struct pci_driver xmgmt_driver = { > > > + .name = XMGMT_MODULE_NAME, > > > + .id_table = xmgmt_pci_ids, > > > + .probe = xmgmt_probe, > > > + .remove = xmgmt_remove, > > > +}; > > > + > > > +static int __init xmgmt_init(void) > > > +{ > > > + int res = 0; > > > + > > > + res = pci_register_driver(&xmgmt_driver); > > > + if (res) > > > + return res; > > > + > > > + return 0; > > > +} > > > + > > > +static __exit void xmgmt_exit(void) > > > +{ > > > + pci_unregister_driver(&xmgmt_driver); > > > +} > > > + > > > +module_init(xmgmt_init); > > > +module_exit(xmgmt_exit); > > > + > > > +MODULE_DEVICE_TABLE(pci, xmgmt_pci_ids); > > > +MODULE_AUTHOR("XRT Team <runtime@xilinx.com>"); > > > +MODULE_DESCRIPTION("Xilinx Alveo management function driver"); > > > +MODULE_LICENSE("GPL v2"); > > > -- > > > 2.27.0
On 1/13/22 5:43 PM, Xu Yilun wrote: > On Thu, Jan 13, 2022 at 03:41:47PM -0800, Lizhi Hou wrote: >> On 1/10/22 11:00 PM, Xu Yilun wrote: >>> = >>> >>> On Wed, Jan 05, 2022 at 02:50:13PM -0800, Lizhi Hou wrote: >>>> The PCIE device driver which attaches to management function on Alveo >>>> devices. It instantiates one or more partition. Each partition consists >>>> a set of hardward endpoints. A flat device tree is associated with each >>>> partition. The first version of this driver uses test version flat device >>>> tree and call xrt lib API to unflatten it. >>>> >>>> Signed-off-by: Sonal Santan <sonal.santan@xilinx.com> >>>> Signed-off-by: Max Zhen <max.zhen@xilinx.com> >>>> Signed-off-by: Lizhi Hou <lizhi.hou@xilinx.com> >>>> --- >>>> drivers/fpga/Makefile | 1 + >>>> drivers/fpga/xrt/Kconfig | 1 + >>>> drivers/fpga/xrt/mgmt/Kconfig | 14 +++ >>>> drivers/fpga/xrt/mgmt/Makefile | 16 +++ >>>> drivers/fpga/xrt/mgmt/dt-test.dts | 12 +++ >>>> drivers/fpga/xrt/mgmt/dt-test.h | 15 +++ >>>> drivers/fpga/xrt/mgmt/xmgmt-drv.c | 158 ++++++++++++++++++++++++++++++ >>>> 7 files changed, 217 insertions(+) >>>> create mode 100644 drivers/fpga/xrt/mgmt/Kconfig >>>> create mode 100644 drivers/fpga/xrt/mgmt/Makefile >>>> create mode 100644 drivers/fpga/xrt/mgmt/dt-test.dts >>>> create mode 100644 drivers/fpga/xrt/mgmt/dt-test.h >>>> create mode 100644 drivers/fpga/xrt/mgmt/xmgmt-drv.c >>>> >>>> diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile >>>> index 5bd41cf4c7ec..544e2144878f 100644 >>>> --- a/drivers/fpga/Makefile >>>> +++ b/drivers/fpga/Makefile >>>> @@ -52,3 +52,4 @@ obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o >>>> >>>> # XRT drivers for Alveo >>>> obj-$(CONFIG_FPGA_XRT_LIB) += xrt/lib/ >>>> +obj-$(CONFIG_FPGA_XRT_XMGMT) += xrt/mgmt/ >>>> diff --git a/drivers/fpga/xrt/Kconfig b/drivers/fpga/xrt/Kconfig >>>> index 04c3bb5aaf4f..50422f77c6df 100644 >>>> --- a/drivers/fpga/xrt/Kconfig >>>> +++ b/drivers/fpga/xrt/Kconfig >>>> @@ -4,3 +4,4 @@ >>>> # >>>> >>>> source "drivers/fpga/xrt/lib/Kconfig" >>>> +source "drivers/fpga/xrt/mgmt/Kconfig" >>>> diff --git a/drivers/fpga/xrt/mgmt/Kconfig b/drivers/fpga/xrt/mgmt/Kconfig >>>> new file mode 100644 >>>> index 000000000000..a978747482be >>>> --- /dev/null >>>> +++ b/drivers/fpga/xrt/mgmt/Kconfig >>>> @@ -0,0 +1,14 @@ >>>> +# SPDX-License-Identifier: GPL-2.0-only >>>> +# >>>> +# Xilinx XRT FPGA device configuration >>>> +# >>>> + >>>> +config FPGA_XRT_XMGMT >>>> + tristate "Xilinx Alveo Management Driver" >>>> + depends on FPGA_XRT_LIB >>>> + select FPGA_BRIDGE >>>> + select FPGA_REGION >>>> + help >>>> + Select this option to enable XRT PCIe driver for Xilinx Alveo FPGA. >>>> + This driver provides interfaces for userspace application to access >>>> + Alveo FPGA device. >>>> diff --git a/drivers/fpga/xrt/mgmt/Makefile b/drivers/fpga/xrt/mgmt/Makefile >>>> new file mode 100644 >>>> index 000000000000..c5134bf71cca >>>> --- /dev/null >>>> +++ b/drivers/fpga/xrt/mgmt/Makefile >>>> @@ -0,0 +1,16 @@ >>>> +# SPDX-License-Identifier: GPL-2.0 >>>> +# >>>> +# Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved. >>>> +# >>>> +# Authors: Sonal.Santan@xilinx.com >>>> +# >>>> + >>>> +FULL_XRT_PATH=$(srctree)/$(src)/.. >>>> + >>>> +obj-$(CONFIG_FPGA_XRT_LIB) += xrt-mgmt.o >>>> + >>>> +xrt-mgmt-objs := \ >>>> + xmgmt-drv.o \ >>>> + dt-test.dtb.o >>>> + >>>> +ccflags-y := -I$(FULL_XRT_PATH)/include >>>> diff --git a/drivers/fpga/xrt/mgmt/dt-test.dts b/drivers/fpga/xrt/mgmt/dt-test.dts >>>> new file mode 100644 >>>> index 000000000000..68dbcb7fd79d >>>> --- /dev/null >>>> +++ b/drivers/fpga/xrt/mgmt/dt-test.dts >>>> @@ -0,0 +1,12 @@ >>>> +// SPDX-License-Identifier: GPL-2.0 >>>> +/dts-v1/; >>>> + >>>> +/ { >>>> + compatible = "xlnx,alveo-partition", "simple-bus"; >>>> + #address-cells = <2>; >>>> + #size-cells = <2>; >>>> + pr_isolate_ulp@0,41000 { >>>> + compatible = "xlnx,alveo-pr-isolation"; >>>> + reg = <0x0 0x41000 0x0 0x1000>; >>>> + }; >>>> +}; >>> I remember Rob's comments: >>> >>> "we'd need to create a base tree (if there isn't one) with nodes >>> for the USB or PCI device(s) and then an overlay for the device can be >>> applied to those nodes." >>> >>> https://lore.kernel.org/linux-fpga/CAL_JsqJfyRymB=VxLuQqLpep+Q1Eie48dobv9sC5OizDz0d2DQ@mail.gmail.com/ >>> >>> So could we firstly create a pci device node under the of_root when >>> the driver probing, then add the partition DTs under the pci device node >>> by overlay machenism. >>> >>> I'm considering if finally we could leverage the existing of-fpga-region >>> that reprograms and enumerates FPGA sub devices by overlay. >>> >>> Open for discussion. >> I would list the main ideas of our implementation >> >> 1. “alveo-partition (simple-bus)” node is created under of_root in xmgmt >> probe function. “simple-bus” is used as it fits well and we can modify it to >> “fpga-region” when we refactor fpga-mgr/region/bridge part. > OK. "fpga-region" for an independent reprogramable region. And > "simple-bus" for static board components. > >> 2. we are the first pcie device using flatten device tree to describe sub >> devices. Adding device tree nodes for all pcie device sounds intrusive. We >> can expend to all pcie device when we have another user of this feature. > I don't mean we create DT nodes for all pcie devices in system, it is the > device driver's choice to create the node for DT style enumeration of its > sub devices. This is mainly for pcie based fpga cards. Now there exsits Glad to know this. > plenty of these cards in the world, and the enumeration of their sub > devices could be a common concern. I think it's good we have a clear > design at the beginning. For single Alveo card on system, we are thinking this structure of_root -----alveo-part@0 (simple-bus or fpga-region) |---alveo-part@1 (fpga-region or fpga-region) ..... Are you suggesting to add a pcie device layer. e.g. of_root----alveo-fpga@<addr>----alveo-part@0 |--alveo-part@1 Thanks, Lizhi > >> 3. of_overlay_fdt_apply() does not support overlaying to a dynamic generated > Yes that's the problem we have. > >> parent node. Thus, we use of_dynamic functions of_changeset_attach_node() >> instead. (similar to drivers/pci/hotplug/pnv_php.c,) . If needed ,we can >> consider to add parameter “parent node” to of_overlay_fdt_apply() to support >> dynamical overlay case. > I agree we do some investigation for this. > > Thanks, > Yilun > >> >> Thanks, >> >> Lizhi >> >>> Thanks, >>> Yilun >>> >>>> diff --git a/drivers/fpga/xrt/mgmt/dt-test.h b/drivers/fpga/xrt/mgmt/dt-test.h >>>> new file mode 100644 >>>> index 000000000000..6ec4203afbd2 >>>> --- /dev/null >>>> +++ b/drivers/fpga/xrt/mgmt/dt-test.h >>>> @@ -0,0 +1,15 @@ >>>> +/* SPDX-License-Identifier: GPL-2.0 */ >>>> +/* >>>> + * Copyright (C) 2020-2022 Xilinx, Inc. >>>> + * >>>> + * Authors: >>>> + * Lizhi Hou <lizhih@xilinx.com> >>>> + */ >>>> + >>>> +#ifndef _DT_TEST_H_ >>>> +#define _DT_TEST_H_ >>>> + >>>> +extern u8 __dtb_dt_test_begin[]; >>>> +extern u8 __dtb_dt_test_end[]; >>>> + >>>> +#endif /* _DT_TEST_H_ */ >>>> diff --git a/drivers/fpga/xrt/mgmt/xmgmt-drv.c b/drivers/fpga/xrt/mgmt/xmgmt-drv.c >>>> new file mode 100644 >>>> index 000000000000..87abe5b86e0b >>>> --- /dev/null >>>> +++ b/drivers/fpga/xrt/mgmt/xmgmt-drv.c >>>> @@ -0,0 +1,158 @@ >>>> +// SPDX-License-Identifier: GPL-2.0 >>>> +/* >>>> + * Xilinx Alveo Management Function Driver >>>> + * >>>> + * Copyright (C) 2020-2022 Xilinx, Inc. >>>> + * >>>> + * Authors: >>>> + * Cheng Zhen <maxz@xilinx.com> >>>> + * Lizhi Hou <lizhih@xilinx.com> >>>> + */ >>>> + >>>> +#include <linux/module.h> >>>> +#include <linux/pci.h> >>>> +#include <linux/aer.h> >>>> +#include <linux/vmalloc.h> >>>> +#include <linux/delay.h> >>>> +#include "xpartition.h" >>>> +#include "dt-test.h" >>>> + >>>> +#define XMGMT_MODULE_NAME "xrt-mgmt" >>>> + >>>> +#define XMGMT_PDEV(xm) ((xm)->pdev) >>>> +#define XMGMT_DEV(xm) (&(XMGMT_PDEV(xm)->dev)) >>>> +#define xmgmt_err(xm, fmt, args...) \ >>>> + dev_err(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >>>> +#define xmgmt_warn(xm, fmt, args...) \ >>>> + dev_warn(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >>>> +#define xmgmt_info(xm, fmt, args...) \ >>>> + dev_info(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >>>> +#define xmgmt_dbg(xm, fmt, args...) \ >>>> + dev_dbg(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) >>>> +#define XMGMT_DEV_ID(_pcidev) \ >>>> + ({ typeof(_pcidev) (pcidev) = (_pcidev); \ >>>> + ((pci_domain_nr((pcidev)->bus) << 16) | \ >>>> + PCI_DEVID((pcidev)->bus->number, (pcidev)->devfn)); }) >>>> + >>>> +#define XRT_MAX_READRQ 512 >>>> + >>>> +/* PCI Device IDs */ >>>> +#define PCI_DEVICE_ID_U50 0x5020 >>>> +static const struct pci_device_id xmgmt_pci_ids[] = { >>>> + { PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_U50), }, /* Alveo U50 */ >>>> + { 0, } >>>> +}; >>>> + >>>> +struct xmgmt { >>>> + struct pci_dev *pdev; >>>> + void *base_partition; >>>> + >>>> + bool ready; >>>> +}; >>>> + >>>> +static int xmgmt_config_pci(struct xmgmt *xm) >>>> +{ >>>> + struct pci_dev *pdev = XMGMT_PDEV(xm); >>>> + int rc; >>>> + >>>> + rc = pcim_enable_device(pdev); >>>> + if (rc < 0) { >>>> + xmgmt_err(xm, "failed to enable device: %d", rc); >>>> + return rc; >>>> + } >>>> + >>>> + rc = pci_enable_pcie_error_reporting(pdev); >>>> + if (rc) >>>> + xmgmt_warn(xm, "failed to enable AER: %d", rc); >>>> + >>>> + pci_set_master(pdev); >>>> + >>>> + rc = pcie_get_readrq(pdev); >>>> + if (rc > XRT_MAX_READRQ) >>>> + pcie_set_readrq(pdev, XRT_MAX_READRQ); >>>> + return 0; >>>> +} >>>> + >>>> +static int xmgmt_probe(struct pci_dev *pdev, const struct pci_device_id *id) >>>> +{ >>>> + struct xrt_partition_range ranges[PCI_NUM_RESOURCES]; >>>> + struct xrt_partition_info xp_info = { 0 }; >>>> + struct device *dev = &pdev->dev; >>>> + int ret, i, idx = 0; >>>> + struct xmgmt *xm; >>>> + >>>> + xm = devm_kzalloc(dev, sizeof(*xm), GFP_KERNEL); >>>> + if (!xm) >>>> + return -ENOMEM; >>>> + xm->pdev = pdev; >>>> + pci_set_drvdata(pdev, xm); >>>> + >>>> + ret = xmgmt_config_pci(xm); >>>> + if (ret) >>>> + goto failed; >>>> + >>>> + for (i = 0; i < PCI_NUM_RESOURCES; i++) { >>>> + if (pci_resource_len(pdev, i) > 0) { >>>> + ranges[idx].bar_idx = i; >>>> + ranges[idx].base = pci_resource_start(pdev, i); >>>> + ranges[idx].size = pci_resource_len(pdev, i); >>>> + idx++; >>>> + } >>>> + } >>>> + xp_info.num_range = idx; >>>> + xp_info.ranges = ranges; >>>> + xp_info.fdt = __dtb_dt_test_begin; >>>> + xp_info.fdt_len = (u32)(__dtb_dt_test_end - __dtb_dt_test_begin); >>>> + ret = xrt_partition_create(&pdev->dev, &xp_info, &xm->base_partition); >>>> + if (ret) >>>> + goto failed; >>>> + >>>> + xmgmt_info(xm, "%s started successfully", XMGMT_MODULE_NAME); >>>> + return 0; >>>> + >>>> +failed: >>>> + if (xm->base_partition) >>>> + xrt_partition_destroy(xm->base_partition); >>>> + pci_set_drvdata(pdev, NULL); >>>> + return ret; >>>> +} >>>> + >>>> +static void xmgmt_remove(struct pci_dev *pdev) >>>> +{ >>>> + struct xmgmt *xm = pci_get_drvdata(pdev); >>>> + >>>> + xrt_partition_destroy(xm->base_partition); >>>> + pci_disable_pcie_error_reporting(xm->pdev); >>>> + xmgmt_info(xm, "%s cleaned up successfully", XMGMT_MODULE_NAME); >>>> +} >>>> + >>>> +static struct pci_driver xmgmt_driver = { >>>> + .name = XMGMT_MODULE_NAME, >>>> + .id_table = xmgmt_pci_ids, >>>> + .probe = xmgmt_probe, >>>> + .remove = xmgmt_remove, >>>> +}; >>>> + >>>> +static int __init xmgmt_init(void) >>>> +{ >>>> + int res = 0; >>>> + >>>> + res = pci_register_driver(&xmgmt_driver); >>>> + if (res) >>>> + return res; >>>> + >>>> + return 0; >>>> +} >>>> + >>>> +static __exit void xmgmt_exit(void) >>>> +{ >>>> + pci_unregister_driver(&xmgmt_driver); >>>> +} >>>> + >>>> +module_init(xmgmt_init); >>>> +module_exit(xmgmt_exit); >>>> + >>>> +MODULE_DEVICE_TABLE(pci, xmgmt_pci_ids); >>>> +MODULE_AUTHOR("XRT Team <runtime@xilinx.com>"); >>>> +MODULE_DESCRIPTION("Xilinx Alveo management function driver"); >>>> +MODULE_LICENSE("GPL v2"); >>>> -- >>>> 2.27.0
On Mon, Jan 17, 2022 at 09:43:40AM -0800, Lizhi Hou wrote: > > On 1/13/22 5:43 PM, Xu Yilun wrote: > > On Thu, Jan 13, 2022 at 03:41:47PM -0800, Lizhi Hou wrote: > > > On 1/10/22 11:00 PM, Xu Yilun wrote: > > > > = > > > > > > > > On Wed, Jan 05, 2022 at 02:50:13PM -0800, Lizhi Hou wrote: > > > > > The PCIE device driver which attaches to management function on Alveo > > > > > devices. It instantiates one or more partition. Each partition consists > > > > > a set of hardward endpoints. A flat device tree is associated with each > > > > > partition. The first version of this driver uses test version flat device > > > > > tree and call xrt lib API to unflatten it. > > > > > > > > > > Signed-off-by: Sonal Santan <sonal.santan@xilinx.com> > > > > > Signed-off-by: Max Zhen <max.zhen@xilinx.com> > > > > > Signed-off-by: Lizhi Hou <lizhi.hou@xilinx.com> > > > > > --- > > > > > drivers/fpga/Makefile | 1 + > > > > > drivers/fpga/xrt/Kconfig | 1 + > > > > > drivers/fpga/xrt/mgmt/Kconfig | 14 +++ > > > > > drivers/fpga/xrt/mgmt/Makefile | 16 +++ > > > > > drivers/fpga/xrt/mgmt/dt-test.dts | 12 +++ > > > > > drivers/fpga/xrt/mgmt/dt-test.h | 15 +++ > > > > > drivers/fpga/xrt/mgmt/xmgmt-drv.c | 158 ++++++++++++++++++++++++++++++ > > > > > 7 files changed, 217 insertions(+) > > > > > create mode 100644 drivers/fpga/xrt/mgmt/Kconfig > > > > > create mode 100644 drivers/fpga/xrt/mgmt/Makefile > > > > > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.dts > > > > > create mode 100644 drivers/fpga/xrt/mgmt/dt-test.h > > > > > create mode 100644 drivers/fpga/xrt/mgmt/xmgmt-drv.c > > > > > > > > > > diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile > > > > > index 5bd41cf4c7ec..544e2144878f 100644 > > > > > --- a/drivers/fpga/Makefile > > > > > +++ b/drivers/fpga/Makefile > > > > > @@ -52,3 +52,4 @@ obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o > > > > > > > > > > # XRT drivers for Alveo > > > > > obj-$(CONFIG_FPGA_XRT_LIB) += xrt/lib/ > > > > > +obj-$(CONFIG_FPGA_XRT_XMGMT) += xrt/mgmt/ > > > > > diff --git a/drivers/fpga/xrt/Kconfig b/drivers/fpga/xrt/Kconfig > > > > > index 04c3bb5aaf4f..50422f77c6df 100644 > > > > > --- a/drivers/fpga/xrt/Kconfig > > > > > +++ b/drivers/fpga/xrt/Kconfig > > > > > @@ -4,3 +4,4 @@ > > > > > # > > > > > > > > > > source "drivers/fpga/xrt/lib/Kconfig" > > > > > +source "drivers/fpga/xrt/mgmt/Kconfig" > > > > > diff --git a/drivers/fpga/xrt/mgmt/Kconfig b/drivers/fpga/xrt/mgmt/Kconfig > > > > > new file mode 100644 > > > > > index 000000000000..a978747482be > > > > > --- /dev/null > > > > > +++ b/drivers/fpga/xrt/mgmt/Kconfig > > > > > @@ -0,0 +1,14 @@ > > > > > +# SPDX-License-Identifier: GPL-2.0-only > > > > > +# > > > > > +# Xilinx XRT FPGA device configuration > > > > > +# > > > > > + > > > > > +config FPGA_XRT_XMGMT > > > > > + tristate "Xilinx Alveo Management Driver" > > > > > + depends on FPGA_XRT_LIB > > > > > + select FPGA_BRIDGE > > > > > + select FPGA_REGION > > > > > + help > > > > > + Select this option to enable XRT PCIe driver for Xilinx Alveo FPGA. > > > > > + This driver provides interfaces for userspace application to access > > > > > + Alveo FPGA device. > > > > > diff --git a/drivers/fpga/xrt/mgmt/Makefile b/drivers/fpga/xrt/mgmt/Makefile > > > > > new file mode 100644 > > > > > index 000000000000..c5134bf71cca > > > > > --- /dev/null > > > > > +++ b/drivers/fpga/xrt/mgmt/Makefile > > > > > @@ -0,0 +1,16 @@ > > > > > +# SPDX-License-Identifier: GPL-2.0 > > > > > +# > > > > > +# Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved. > > > > > +# > > > > > +# Authors: Sonal.Santan@xilinx.com > > > > > +# > > > > > + > > > > > +FULL_XRT_PATH=$(srctree)/$(src)/.. > > > > > + > > > > > +obj-$(CONFIG_FPGA_XRT_LIB) += xrt-mgmt.o > > > > > + > > > > > +xrt-mgmt-objs := \ > > > > > + xmgmt-drv.o \ > > > > > + dt-test.dtb.o > > > > > + > > > > > +ccflags-y := -I$(FULL_XRT_PATH)/include > > > > > diff --git a/drivers/fpga/xrt/mgmt/dt-test.dts b/drivers/fpga/xrt/mgmt/dt-test.dts > > > > > new file mode 100644 > > > > > index 000000000000..68dbcb7fd79d > > > > > --- /dev/null > > > > > +++ b/drivers/fpga/xrt/mgmt/dt-test.dts > > > > > @@ -0,0 +1,12 @@ > > > > > +// SPDX-License-Identifier: GPL-2.0 > > > > > +/dts-v1/; > > > > > + > > > > > +/ { > > > > > + compatible = "xlnx,alveo-partition", "simple-bus"; > > > > > + #address-cells = <2>; > > > > > + #size-cells = <2>; > > > > > + pr_isolate_ulp@0,41000 { > > > > > + compatible = "xlnx,alveo-pr-isolation"; > > > > > + reg = <0x0 0x41000 0x0 0x1000>; > > > > > + }; > > > > > +}; > > > > I remember Rob's comments: > > > > > > > > "we'd need to create a base tree (if there isn't one) with nodes > > > > for the USB or PCI device(s) and then an overlay for the device can be > > > > applied to those nodes." > > > > > > > > https://lore.kernel.org/linux-fpga/CAL_JsqJfyRymB=VxLuQqLpep+Q1Eie48dobv9sC5OizDz0d2DQ@mail.gmail.com/ > > > > > > > > So could we firstly create a pci device node under the of_root when > > > > the driver probing, then add the partition DTs under the pci device node > > > > by overlay machenism. > > > > > > > > I'm considering if finally we could leverage the existing of-fpga-region > > > > that reprograms and enumerates FPGA sub devices by overlay. > > > > > > > > Open for discussion. > > > I would list the main ideas of our implementation > > > > > > 1. “alveo-partition (simple-bus)” node is created under of_root in xmgmt > > > probe function. “simple-bus” is used as it fits well and we can modify it to > > > “fpga-region” when we refactor fpga-mgr/region/bridge part. > > OK. "fpga-region" for an independent reprogramable region. And > > "simple-bus" for static board components. > > > > > 2. we are the first pcie device using flatten device tree to describe sub > > > devices. Adding device tree nodes for all pcie device sounds intrusive. We > > > can expend to all pcie device when we have another user of this feature. > > I don't mean we create DT nodes for all pcie devices in system, it is the > > device driver's choice to create the node for DT style enumeration of its > > sub devices. This is mainly for pcie based fpga cards. Now there exsits > Glad to know this. > > plenty of these cards in the world, and the enumeration of their sub > > devices could be a common concern. I think it's good we have a clear > > design at the beginning. > > For single Alveo card on system, we are thinking this structure > > of_root -----alveo-part@0 (simple-bus or fpga-region) > > |---alveo-part@1 (fpga-region or fpga-region) > > ..... > > Are you suggesting to add a pcie device layer. e.g. > > of_root----alveo-fpga@<addr>----alveo-part@0 > > |--alveo-part@1 > Yes, my idea is that we don't build the whole PCI/e tree (RPs, Bridges, switches). Instead we add a PCI/e DT node directly under of_root if the driver wants to append its DT overlay for its sub devices. There could be changes to common pci/of or DT domain, so for sure we need to discuss it with the PCI/DT maintainers. Thanks, Yilun > > Thanks, > > Lizhi > > > > > > 3. of_overlay_fdt_apply() does not support overlaying to a dynamic generated > > Yes that's the problem we have. > > > > > parent node. Thus, we use of_dynamic functions of_changeset_attach_node() > > > instead. (similar to drivers/pci/hotplug/pnv_php.c,) . If needed ,we can > > > consider to add parameter “parent node” to of_overlay_fdt_apply() to support > > > dynamical overlay case. > > I agree we do some investigation for this. > > > > Thanks, > > Yilun > > > > > > > > Thanks, > > > > > > Lizhi > > > > > > > Thanks, > > > > Yilun > > > > > > > > > diff --git a/drivers/fpga/xrt/mgmt/dt-test.h b/drivers/fpga/xrt/mgmt/dt-test.h > > > > > new file mode 100644 > > > > > index 000000000000..6ec4203afbd2 > > > > > --- /dev/null > > > > > +++ b/drivers/fpga/xrt/mgmt/dt-test.h > > > > > @@ -0,0 +1,15 @@ > > > > > +/* SPDX-License-Identifier: GPL-2.0 */ > > > > > +/* > > > > > + * Copyright (C) 2020-2022 Xilinx, Inc. > > > > > + * > > > > > + * Authors: > > > > > + * Lizhi Hou <lizhih@xilinx.com> > > > > > + */ > > > > > + > > > > > +#ifndef _DT_TEST_H_ > > > > > +#define _DT_TEST_H_ > > > > > + > > > > > +extern u8 __dtb_dt_test_begin[]; > > > > > +extern u8 __dtb_dt_test_end[]; > > > > > + > > > > > +#endif /* _DT_TEST_H_ */ > > > > > diff --git a/drivers/fpga/xrt/mgmt/xmgmt-drv.c b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > > > > > new file mode 100644 > > > > > index 000000000000..87abe5b86e0b > > > > > --- /dev/null > > > > > +++ b/drivers/fpga/xrt/mgmt/xmgmt-drv.c > > > > > @@ -0,0 +1,158 @@ > > > > > +// SPDX-License-Identifier: GPL-2.0 > > > > > +/* > > > > > + * Xilinx Alveo Management Function Driver > > > > > + * > > > > > + * Copyright (C) 2020-2022 Xilinx, Inc. > > > > > + * > > > > > + * Authors: > > > > > + * Cheng Zhen <maxz@xilinx.com> > > > > > + * Lizhi Hou <lizhih@xilinx.com> > > > > > + */ > > > > > + > > > > > +#include <linux/module.h> > > > > > +#include <linux/pci.h> > > > > > +#include <linux/aer.h> > > > > > +#include <linux/vmalloc.h> > > > > > +#include <linux/delay.h> > > > > > +#include "xpartition.h" > > > > > +#include "dt-test.h" > > > > > + > > > > > +#define XMGMT_MODULE_NAME "xrt-mgmt" > > > > > + > > > > > +#define XMGMT_PDEV(xm) ((xm)->pdev) > > > > > +#define XMGMT_DEV(xm) (&(XMGMT_PDEV(xm)->dev)) > > > > > +#define xmgmt_err(xm, fmt, args...) \ > > > > > + dev_err(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > > > +#define xmgmt_warn(xm, fmt, args...) \ > > > > > + dev_warn(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > > > +#define xmgmt_info(xm, fmt, args...) \ > > > > > + dev_info(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > > > +#define xmgmt_dbg(xm, fmt, args...) \ > > > > > + dev_dbg(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) > > > > > +#define XMGMT_DEV_ID(_pcidev) \ > > > > > + ({ typeof(_pcidev) (pcidev) = (_pcidev); \ > > > > > + ((pci_domain_nr((pcidev)->bus) << 16) | \ > > > > > + PCI_DEVID((pcidev)->bus->number, (pcidev)->devfn)); }) > > > > > + > > > > > +#define XRT_MAX_READRQ 512 > > > > > + > > > > > +/* PCI Device IDs */ > > > > > +#define PCI_DEVICE_ID_U50 0x5020 > > > > > +static const struct pci_device_id xmgmt_pci_ids[] = { > > > > > + { PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_U50), }, /* Alveo U50 */ > > > > > + { 0, } > > > > > +}; > > > > > + > > > > > +struct xmgmt { > > > > > + struct pci_dev *pdev; > > > > > + void *base_partition; > > > > > + > > > > > + bool ready; > > > > > +}; > > > > > + > > > > > +static int xmgmt_config_pci(struct xmgmt *xm) > > > > > +{ > > > > > + struct pci_dev *pdev = XMGMT_PDEV(xm); > > > > > + int rc; > > > > > + > > > > > + rc = pcim_enable_device(pdev); > > > > > + if (rc < 0) { > > > > > + xmgmt_err(xm, "failed to enable device: %d", rc); > > > > > + return rc; > > > > > + } > > > > > + > > > > > + rc = pci_enable_pcie_error_reporting(pdev); > > > > > + if (rc) > > > > > + xmgmt_warn(xm, "failed to enable AER: %d", rc); > > > > > + > > > > > + pci_set_master(pdev); > > > > > + > > > > > + rc = pcie_get_readrq(pdev); > > > > > + if (rc > XRT_MAX_READRQ) > > > > > + pcie_set_readrq(pdev, XRT_MAX_READRQ); > > > > > + return 0; > > > > > +} > > > > > + > > > > > +static int xmgmt_probe(struct pci_dev *pdev, const struct pci_device_id *id) > > > > > +{ > > > > > + struct xrt_partition_range ranges[PCI_NUM_RESOURCES]; > > > > > + struct xrt_partition_info xp_info = { 0 }; > > > > > + struct device *dev = &pdev->dev; > > > > > + int ret, i, idx = 0; > > > > > + struct xmgmt *xm; > > > > > + > > > > > + xm = devm_kzalloc(dev, sizeof(*xm), GFP_KERNEL); > > > > > + if (!xm) > > > > > + return -ENOMEM; > > > > > + xm->pdev = pdev; > > > > > + pci_set_drvdata(pdev, xm); > > > > > + > > > > > + ret = xmgmt_config_pci(xm); > > > > > + if (ret) > > > > > + goto failed; > > > > > + > > > > > + for (i = 0; i < PCI_NUM_RESOURCES; i++) { > > > > > + if (pci_resource_len(pdev, i) > 0) { > > > > > + ranges[idx].bar_idx = i; > > > > > + ranges[idx].base = pci_resource_start(pdev, i); > > > > > + ranges[idx].size = pci_resource_len(pdev, i); > > > > > + idx++; > > > > > + } > > > > > + } > > > > > + xp_info.num_range = idx; > > > > > + xp_info.ranges = ranges; > > > > > + xp_info.fdt = __dtb_dt_test_begin; > > > > > + xp_info.fdt_len = (u32)(__dtb_dt_test_end - __dtb_dt_test_begin); > > > > > + ret = xrt_partition_create(&pdev->dev, &xp_info, &xm->base_partition); > > > > > + if (ret) > > > > > + goto failed; > > > > > + > > > > > + xmgmt_info(xm, "%s started successfully", XMGMT_MODULE_NAME); > > > > > + return 0; > > > > > + > > > > > +failed: > > > > > + if (xm->base_partition) > > > > > + xrt_partition_destroy(xm->base_partition); > > > > > + pci_set_drvdata(pdev, NULL); > > > > > + return ret; > > > > > +} > > > > > + > > > > > +static void xmgmt_remove(struct pci_dev *pdev) > > > > > +{ > > > > > + struct xmgmt *xm = pci_get_drvdata(pdev); > > > > > + > > > > > + xrt_partition_destroy(xm->base_partition); > > > > > + pci_disable_pcie_error_reporting(xm->pdev); > > > > > + xmgmt_info(xm, "%s cleaned up successfully", XMGMT_MODULE_NAME); > > > > > +} > > > > > + > > > > > +static struct pci_driver xmgmt_driver = { > > > > > + .name = XMGMT_MODULE_NAME, > > > > > + .id_table = xmgmt_pci_ids, > > > > > + .probe = xmgmt_probe, > > > > > + .remove = xmgmt_remove, > > > > > +}; > > > > > + > > > > > +static int __init xmgmt_init(void) > > > > > +{ > > > > > + int res = 0; > > > > > + > > > > > + res = pci_register_driver(&xmgmt_driver); > > > > > + if (res) > > > > > + return res; > > > > > + > > > > > + return 0; > > > > > +} > > > > > + > > > > > +static __exit void xmgmt_exit(void) > > > > > +{ > > > > > + pci_unregister_driver(&xmgmt_driver); > > > > > +} > > > > > + > > > > > +module_init(xmgmt_init); > > > > > +module_exit(xmgmt_exit); > > > > > + > > > > > +MODULE_DEVICE_TABLE(pci, xmgmt_pci_ids); > > > > > +MODULE_AUTHOR("XRT Team <runtime@xilinx.com>"); > > > > > +MODULE_DESCRIPTION("Xilinx Alveo management function driver"); > > > > > +MODULE_LICENSE("GPL v2"); > > > > > -- > > > > > 2.27.0
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile index 5bd41cf4c7ec..544e2144878f 100644 --- a/drivers/fpga/Makefile +++ b/drivers/fpga/Makefile @@ -52,3 +52,4 @@ obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o # XRT drivers for Alveo obj-$(CONFIG_FPGA_XRT_LIB) += xrt/lib/ +obj-$(CONFIG_FPGA_XRT_XMGMT) += xrt/mgmt/ diff --git a/drivers/fpga/xrt/Kconfig b/drivers/fpga/xrt/Kconfig index 04c3bb5aaf4f..50422f77c6df 100644 --- a/drivers/fpga/xrt/Kconfig +++ b/drivers/fpga/xrt/Kconfig @@ -4,3 +4,4 @@ # source "drivers/fpga/xrt/lib/Kconfig" +source "drivers/fpga/xrt/mgmt/Kconfig" diff --git a/drivers/fpga/xrt/mgmt/Kconfig b/drivers/fpga/xrt/mgmt/Kconfig new file mode 100644 index 000000000000..a978747482be --- /dev/null +++ b/drivers/fpga/xrt/mgmt/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Xilinx XRT FPGA device configuration +# + +config FPGA_XRT_XMGMT + tristate "Xilinx Alveo Management Driver" + depends on FPGA_XRT_LIB + select FPGA_BRIDGE + select FPGA_REGION + help + Select this option to enable XRT PCIe driver for Xilinx Alveo FPGA. + This driver provides interfaces for userspace application to access + Alveo FPGA device. diff --git a/drivers/fpga/xrt/mgmt/Makefile b/drivers/fpga/xrt/mgmt/Makefile new file mode 100644 index 000000000000..c5134bf71cca --- /dev/null +++ b/drivers/fpga/xrt/mgmt/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2020-2022 Xilinx, Inc. All rights reserved. +# +# Authors: Sonal.Santan@xilinx.com +# + +FULL_XRT_PATH=$(srctree)/$(src)/.. + +obj-$(CONFIG_FPGA_XRT_LIB) += xrt-mgmt.o + +xrt-mgmt-objs := \ + xmgmt-drv.o \ + dt-test.dtb.o + +ccflags-y := -I$(FULL_XRT_PATH)/include diff --git a/drivers/fpga/xrt/mgmt/dt-test.dts b/drivers/fpga/xrt/mgmt/dt-test.dts new file mode 100644 index 000000000000..68dbcb7fd79d --- /dev/null +++ b/drivers/fpga/xrt/mgmt/dt-test.dts @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/dts-v1/; + +/ { + compatible = "xlnx,alveo-partition", "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + pr_isolate_ulp@0,41000 { + compatible = "xlnx,alveo-pr-isolation"; + reg = <0x0 0x41000 0x0 0x1000>; + }; +}; diff --git a/drivers/fpga/xrt/mgmt/dt-test.h b/drivers/fpga/xrt/mgmt/dt-test.h new file mode 100644 index 000000000000..6ec4203afbd2 --- /dev/null +++ b/drivers/fpga/xrt/mgmt/dt-test.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Xilinx, Inc. + * + * Authors: + * Lizhi Hou <lizhih@xilinx.com> + */ + +#ifndef _DT_TEST_H_ +#define _DT_TEST_H_ + +extern u8 __dtb_dt_test_begin[]; +extern u8 __dtb_dt_test_end[]; + +#endif /* _DT_TEST_H_ */ diff --git a/drivers/fpga/xrt/mgmt/xmgmt-drv.c b/drivers/fpga/xrt/mgmt/xmgmt-drv.c new file mode 100644 index 000000000000..87abe5b86e0b --- /dev/null +++ b/drivers/fpga/xrt/mgmt/xmgmt-drv.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx Alveo Management Function Driver + * + * Copyright (C) 2020-2022 Xilinx, Inc. + * + * Authors: + * Cheng Zhen <maxz@xilinx.com> + * Lizhi Hou <lizhih@xilinx.com> + */ + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/aer.h> +#include <linux/vmalloc.h> +#include <linux/delay.h> +#include "xpartition.h" +#include "dt-test.h" + +#define XMGMT_MODULE_NAME "xrt-mgmt" + +#define XMGMT_PDEV(xm) ((xm)->pdev) +#define XMGMT_DEV(xm) (&(XMGMT_PDEV(xm)->dev)) +#define xmgmt_err(xm, fmt, args...) \ + dev_err(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) +#define xmgmt_warn(xm, fmt, args...) \ + dev_warn(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) +#define xmgmt_info(xm, fmt, args...) \ + dev_info(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) +#define xmgmt_dbg(xm, fmt, args...) \ + dev_dbg(XMGMT_DEV(xm), "%s: " fmt, __func__, ##args) +#define XMGMT_DEV_ID(_pcidev) \ + ({ typeof(_pcidev) (pcidev) = (_pcidev); \ + ((pci_domain_nr((pcidev)->bus) << 16) | \ + PCI_DEVID((pcidev)->bus->number, (pcidev)->devfn)); }) + +#define XRT_MAX_READRQ 512 + +/* PCI Device IDs */ +#define PCI_DEVICE_ID_U50 0x5020 +static const struct pci_device_id xmgmt_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_XILINX, PCI_DEVICE_ID_U50), }, /* Alveo U50 */ + { 0, } +}; + +struct xmgmt { + struct pci_dev *pdev; + void *base_partition; + + bool ready; +}; + +static int xmgmt_config_pci(struct xmgmt *xm) +{ + struct pci_dev *pdev = XMGMT_PDEV(xm); + int rc; + + rc = pcim_enable_device(pdev); + if (rc < 0) { + xmgmt_err(xm, "failed to enable device: %d", rc); + return rc; + } + + rc = pci_enable_pcie_error_reporting(pdev); + if (rc) + xmgmt_warn(xm, "failed to enable AER: %d", rc); + + pci_set_master(pdev); + + rc = pcie_get_readrq(pdev); + if (rc > XRT_MAX_READRQ) + pcie_set_readrq(pdev, XRT_MAX_READRQ); + return 0; +} + +static int xmgmt_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct xrt_partition_range ranges[PCI_NUM_RESOURCES]; + struct xrt_partition_info xp_info = { 0 }; + struct device *dev = &pdev->dev; + int ret, i, idx = 0; + struct xmgmt *xm; + + xm = devm_kzalloc(dev, sizeof(*xm), GFP_KERNEL); + if (!xm) + return -ENOMEM; + xm->pdev = pdev; + pci_set_drvdata(pdev, xm); + + ret = xmgmt_config_pci(xm); + if (ret) + goto failed; + + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + if (pci_resource_len(pdev, i) > 0) { + ranges[idx].bar_idx = i; + ranges[idx].base = pci_resource_start(pdev, i); + ranges[idx].size = pci_resource_len(pdev, i); + idx++; + } + } + xp_info.num_range = idx; + xp_info.ranges = ranges; + xp_info.fdt = __dtb_dt_test_begin; + xp_info.fdt_len = (u32)(__dtb_dt_test_end - __dtb_dt_test_begin); + ret = xrt_partition_create(&pdev->dev, &xp_info, &xm->base_partition); + if (ret) + goto failed; + + xmgmt_info(xm, "%s started successfully", XMGMT_MODULE_NAME); + return 0; + +failed: + if (xm->base_partition) + xrt_partition_destroy(xm->base_partition); + pci_set_drvdata(pdev, NULL); + return ret; +} + +static void xmgmt_remove(struct pci_dev *pdev) +{ + struct xmgmt *xm = pci_get_drvdata(pdev); + + xrt_partition_destroy(xm->base_partition); + pci_disable_pcie_error_reporting(xm->pdev); + xmgmt_info(xm, "%s cleaned up successfully", XMGMT_MODULE_NAME); +} + +static struct pci_driver xmgmt_driver = { + .name = XMGMT_MODULE_NAME, + .id_table = xmgmt_pci_ids, + .probe = xmgmt_probe, + .remove = xmgmt_remove, +}; + +static int __init xmgmt_init(void) +{ + int res = 0; + + res = pci_register_driver(&xmgmt_driver); + if (res) + return res; + + return 0; +} + +static __exit void xmgmt_exit(void) +{ + pci_unregister_driver(&xmgmt_driver); +} + +module_init(xmgmt_init); +module_exit(xmgmt_exit); + +MODULE_DEVICE_TABLE(pci, xmgmt_pci_ids); +MODULE_AUTHOR("XRT Team <runtime@xilinx.com>"); +MODULE_DESCRIPTION("Xilinx Alveo management function driver"); +MODULE_LICENSE("GPL v2");