[v2] PCI: qcom: Add support for modular builds

Commit Message

Johan Hovold July 21, 2022, 6:47 a.m. UTC

Allow the Qualcomm PCIe controller driver to be built as a module, which
is useful for multi-platform kernels as well as during development.

Reviewed-by: Rob Herring <[e-mail]>
Reviewed-by: Manivannan Sadhasivam <[e-mail]>
Signed-off-by: Johan Hovold <[e-mail]>
---

Changes in v2
 - rebase on next-20220720 (adjust context)
 - add Rob and Mani's reviewed-by tags


 drivers/pci/controller/dwc/Kconfig     |  2 +-
 drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
 2 files changed, 34 insertions(+), 4 deletions(-)

Comments

Bjorn Helgaas July 21, 2022, 7:35 p.m. UTC | #1

On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> Allow the Qualcomm PCIe controller driver to be built as a module, which
> is useful for multi-platform kernels as well as during development.
> 
> Reviewed-by: Rob Herring <[e-mail]>
> Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> Signed-off-by: Johan Hovold <[e-mail]>

Applied to pci/ctrl/qcom for v5.20, thanks!

> ---
> 
> Changes in v2
>  - rebase on next-20220720 (adjust context)
>  - add Rob and Mani's reviewed-by tags
> 
> 
>  drivers/pci/controller/dwc/Kconfig     |  2 +-
>  drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
>  2 files changed, 34 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
> index 62ce3abf0f19..230f56d1a268 100644
> --- a/drivers/pci/controller/dwc/Kconfig
> +++ b/drivers/pci/controller/dwc/Kconfig
> @@ -168,7 +168,7 @@ config PCI_HISI
>  	  Hip05 and Hip06 SoCs
>  
>  config PCIE_QCOM
> -	bool "Qualcomm PCIe controller"
> +	tristate "Qualcomm PCIe controller"
>  	depends on OF && (ARCH_QCOM || COMPILE_TEST)
>  	depends on PCI_MSI_IRQ_DOMAIN
>  	select PCIE_DW_HOST
> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> index 5ed164c2afa3..d176c635016b 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> @@ -16,7 +16,7 @@
>  #include <linux/io.h>
>  #include <linux/iopoll.h>
>  #include <linux/kernel.h>
> -#include <linux/init.h>
> +#include <linux/module.h>
>  #include <linux/of_device.h>
>  #include <linux/of_gpio.h>
>  #include <linux/pci.h>
> @@ -1518,6 +1518,15 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
>  	return ret;
>  }
>  
> +static void qcom_pcie_host_deinit(struct qcom_pcie *pcie)
> +{
> +	qcom_ep_reset_assert(pcie);
> +	if (pcie->cfg->ops->post_deinit)
> +		pcie->cfg->ops->post_deinit(pcie);
> +	phy_power_off(pcie->phy);
> +	pcie->cfg->ops->deinit(pcie);
> +}
> +
>  static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
>  	.host_init = qcom_pcie_host_init,
>  };
> @@ -1752,6 +1761,22 @@ static int qcom_pcie_probe(struct platform_device *pdev)
>  	return ret;
>  }
>  
> +static int qcom_pcie_remove(struct platform_device *pdev)
> +{
> +	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
> +	struct device *dev = &pdev->dev;
> +
> +	dw_pcie_host_deinit(&pcie->pci->pp);
> +	qcom_pcie_host_deinit(pcie);
> +
> +	phy_exit(pcie->phy);
> +
> +	pm_runtime_put_sync(dev);
> +	pm_runtime_disable(dev);
> +
> +	return 0;
> +}
> +
>  static const struct of_device_id qcom_pcie_match[] = {
>  	{ .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
>  	{ .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
> @@ -1771,6 +1796,7 @@ static const struct of_device_id qcom_pcie_match[] = {
>  	{ .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
>  	{ }
>  };
> +MODULE_DEVICE_TABLE(of, qcom_pcie_match);
>  
>  static void qcom_fixup_class(struct pci_dev *dev)
>  {
> @@ -1786,10 +1812,14 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
>  
>  static struct platform_driver qcom_pcie_driver = {
>  	.probe = qcom_pcie_probe,
> +	.remove = qcom_pcie_remove,
>  	.driver = {
>  		.name = "qcom-pcie",
> -		.suppress_bind_attrs = true,
>  		.of_match_table = qcom_pcie_match,
>  	},
>  };
> -builtin_platform_driver(qcom_pcie_driver);
> +module_platform_driver(qcom_pcie_driver);
> +
> +MODULE_AUTHOR("Stanimir Varbanov <[e-mail]>");
> +MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
> +MODULE_LICENSE("GPL");
> -- 
> 2.35.1
>

Bjorn Helgaas July 27, 2022, 8:02 p.m. UTC | #2

On Thu, Jul 21, 2022 at 02:35:13PM -0500, Bjorn Helgaas wrote:
> On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> > Allow the Qualcomm PCIe controller driver to be built as a module, which
> > is useful for multi-platform kernels as well as during development.
> > 
> > Reviewed-by: Rob Herring <[e-mail]>
> > Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> > Signed-off-by: Johan Hovold <[e-mail]>
> 
> Applied to pci/ctrl/qcom for v5.20, thanks!

I'm going to drop this one for now, since the module vs remove
discussion [1] is still ongoing.

This patch actually makes it both *modular* and *removable*.  I think
the modular part is uncontroversial and valuable by itself.

If you want to just make it modular and *non*-removable, I think that
would be fine and we can add removability next cycle if we think it's
safe.

[1] https://lore.kernel.org/r/20220721195433.GA1747571@bhelgaas

> > Changes in v2
> >  - rebase on next-20220720 (adjust context)
> >  - add Rob and Mani's reviewed-by tags
> > 
> > 
> >  drivers/pci/controller/dwc/Kconfig     |  2 +-
> >  drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
> >  2 files changed, 34 insertions(+), 4 deletions(-)
> > 
> > diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
> > index 62ce3abf0f19..230f56d1a268 100644
> > --- a/drivers/pci/controller/dwc/Kconfig
> > +++ b/drivers/pci/controller/dwc/Kconfig
> > @@ -168,7 +168,7 @@ config PCI_HISI
> >  	  Hip05 and Hip06 SoCs
> >  
> >  config PCIE_QCOM
> > -	bool "Qualcomm PCIe controller"
> > +	tristate "Qualcomm PCIe controller"
> >  	depends on OF && (ARCH_QCOM || COMPILE_TEST)
> >  	depends on PCI_MSI_IRQ_DOMAIN
> >  	select PCIE_DW_HOST
> > diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> > index 5ed164c2afa3..d176c635016b 100644
> > --- a/drivers/pci/controller/dwc/pcie-qcom.c
> > +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> > @@ -16,7 +16,7 @@
> >  #include <linux/io.h>
> >  #include <linux/iopoll.h>
> >  #include <linux/kernel.h>
> > -#include <linux/init.h>
> > +#include <linux/module.h>
> >  #include <linux/of_device.h>
> >  #include <linux/of_gpio.h>
> >  #include <linux/pci.h>
> > @@ -1518,6 +1518,15 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
> >  	return ret;
> >  }
> >  
> > +static void qcom_pcie_host_deinit(struct qcom_pcie *pcie)
> > +{
> > +	qcom_ep_reset_assert(pcie);
> > +	if (pcie->cfg->ops->post_deinit)
> > +		pcie->cfg->ops->post_deinit(pcie);
> > +	phy_power_off(pcie->phy);
> > +	pcie->cfg->ops->deinit(pcie);
> > +}
> > +
> >  static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
> >  	.host_init = qcom_pcie_host_init,
> >  };
> > @@ -1752,6 +1761,22 @@ static int qcom_pcie_probe(struct platform_device *pdev)
> >  	return ret;
> >  }
> >  
> > +static int qcom_pcie_remove(struct platform_device *pdev)
> > +{
> > +	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
> > +	struct device *dev = &pdev->dev;
> > +
> > +	dw_pcie_host_deinit(&pcie->pci->pp);
> > +	qcom_pcie_host_deinit(pcie);
> > +
> > +	phy_exit(pcie->phy);
> > +
> > +	pm_runtime_put_sync(dev);
> > +	pm_runtime_disable(dev);
> > +
> > +	return 0;
> > +}
> > +
> >  static const struct of_device_id qcom_pcie_match[] = {
> >  	{ .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
> >  	{ .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
> > @@ -1771,6 +1796,7 @@ static const struct of_device_id qcom_pcie_match[] = {
> >  	{ .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
> >  	{ }
> >  };
> > +MODULE_DEVICE_TABLE(of, qcom_pcie_match);
> >  
> >  static void qcom_fixup_class(struct pci_dev *dev)
> >  {
> > @@ -1786,10 +1812,14 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
> >  
> >  static struct platform_driver qcom_pcie_driver = {
> >  	.probe = qcom_pcie_probe,
> > +	.remove = qcom_pcie_remove,
> >  	.driver = {
> >  		.name = "qcom-pcie",
> > -		.suppress_bind_attrs = true,
> >  		.of_match_table = qcom_pcie_match,
> >  	},
> >  };
> > -builtin_platform_driver(qcom_pcie_driver);
> > +module_platform_driver(qcom_pcie_driver);
> > +
> > +MODULE_AUTHOR("Stanimir Varbanov <[e-mail]>");
> > +MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
> > +MODULE_LICENSE("GPL");
> > -- 
> > 2.35.1
> >

Johan Hovold July 28, 2022, 12:19 p.m. UTC | #3

On Wed, Jul 27, 2022 at 03:02:57PM -0500, Bjorn Helgaas wrote:
> On Thu, Jul 21, 2022 at 02:35:13PM -0500, Bjorn Helgaas wrote:
> > On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> > > Allow the Qualcomm PCIe controller driver to be built as a module, which
> > > is useful for multi-platform kernels as well as during development.
> > > 
> > > Reviewed-by: Rob Herring <[e-mail]>
> > > Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> > > Signed-off-by: Johan Hovold <[e-mail]>
> > 
> > Applied to pci/ctrl/qcom for v5.20, thanks!
> 
> I'm going to drop this one for now, since the module vs remove
> discussion [1] is still ongoing.
> 
> This patch actually makes it both *modular* and *removable*.  I think
> the modular part is uncontroversial and valuable by itself.
> 
> If you want to just make it modular and *non*-removable, I think that
> would be fine and we can add removability next cycle if we think it's
> safe.
> 
> [1] https://lore.kernel.org/r/20220721195433.GA1747571@bhelgaas

Nah, I prefer we conclude that discussion so that hopefully this can be
merged as is.

Johan

Johan Hovold Sept. 20, 2022, 8:47 a.m. UTC | #4

Hi Lorenzo,

On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> Allow the Qualcomm PCIe controller driver to be built as a module, which
> is useful for multi-platform kernels as well as during development.
> 
> Reviewed-by: Rob Herring <[e-mail]>
> Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> Signed-off-by: Johan Hovold <[e-mail]>
> ---
> 
> Changes in v2
>  - rebase on next-20220720 (adjust context)
>  - add Rob and Mani's reviewed-by tags

Have you had a change to look at this one since you got back from
vacation?

I believe this should be uncontroversial as we already have other
modular dwc drivers and there's no mapping of legacy INTx interrupts
involved.

>  drivers/pci/controller/dwc/Kconfig     |  2 +-
>  drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
>  2 files changed, 34 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
> index 62ce3abf0f19..230f56d1a268 100644
> --- a/drivers/pci/controller/dwc/Kconfig
> +++ b/drivers/pci/controller/dwc/Kconfig
> @@ -168,7 +168,7 @@ config PCI_HISI
>  	  Hip05 and Hip06 SoCs
>  
>  config PCIE_QCOM
> -	bool "Qualcomm PCIe controller"
> +	tristate "Qualcomm PCIe controller"
>  	depends on OF && (ARCH_QCOM || COMPILE_TEST)
>  	depends on PCI_MSI_IRQ_DOMAIN
>  	select PCIE_DW_HOST
> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> index 5ed164c2afa3..d176c635016b 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> @@ -16,7 +16,7 @@
>  #include <linux/io.h>
>  #include <linux/iopoll.h>
>  #include <linux/kernel.h>
> -#include <linux/init.h>
> +#include <linux/module.h>
>  #include <linux/of_device.h>
>  #include <linux/of_gpio.h>
>  #include <linux/pci.h>
> @@ -1518,6 +1518,15 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
>  	return ret;
>  }
>  
> +static void qcom_pcie_host_deinit(struct qcom_pcie *pcie)
> +{
> +	qcom_ep_reset_assert(pcie);
> +	if (pcie->cfg->ops->post_deinit)
> +		pcie->cfg->ops->post_deinit(pcie);
> +	phy_power_off(pcie->phy);
> +	pcie->cfg->ops->deinit(pcie);
> +}
> +
>  static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
>  	.host_init = qcom_pcie_host_init,
>  };
> @@ -1752,6 +1761,22 @@ static int qcom_pcie_probe(struct platform_device *pdev)
>  	return ret;
>  }
>  
> +static int qcom_pcie_remove(struct platform_device *pdev)
> +{
> +	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
> +	struct device *dev = &pdev->dev;
> +
> +	dw_pcie_host_deinit(&pcie->pci->pp);
> +	qcom_pcie_host_deinit(pcie);
> +
> +	phy_exit(pcie->phy);
> +
> +	pm_runtime_put_sync(dev);
> +	pm_runtime_disable(dev);
> +
> +	return 0;
> +}
> +
>  static const struct of_device_id qcom_pcie_match[] = {
>  	{ .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
>  	{ .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
> @@ -1771,6 +1796,7 @@ static const struct of_device_id qcom_pcie_match[] = {
>  	{ .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
>  	{ }
>  };
> +MODULE_DEVICE_TABLE(of, qcom_pcie_match);
>  
>  static void qcom_fixup_class(struct pci_dev *dev)
>  {
> @@ -1786,10 +1812,14 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
>  
>  static struct platform_driver qcom_pcie_driver = {
>  	.probe = qcom_pcie_probe,
> +	.remove = qcom_pcie_remove,
>  	.driver = {
>  		.name = "qcom-pcie",
> -		.suppress_bind_attrs = true,
>  		.of_match_table = qcom_pcie_match,
>  	},
>  };
> -builtin_platform_driver(qcom_pcie_driver);
> +module_platform_driver(qcom_pcie_driver);
> +
> +MODULE_AUTHOR("Stanimir Varbanov <[e-mail]>");
> +MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
> +MODULE_LICENSE("GPL");

Johan

Stanimir Varbanov Sept. 20, 2022, 10:26 a.m. UTC | #5

Hi Johan,

Thanks for the patch.

On 7/21/22 09:47, Johan Hovold wrote:
> Allow the Qualcomm PCIe controller driver to be built as a module, which
> is useful for multi-platform kernels as well as during development.
> 
> Reviewed-by: Rob Herring <[e-mail]>
> Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> Signed-off-by: Johan Hovold <[e-mail]>
> ---
> 
> Changes in v2
>  - rebase on next-20220720 (adjust context)
>  - add Rob and Mani's reviewed-by tags
> 
> 
>  drivers/pci/controller/dwc/Kconfig     |  2 +-
>  drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
>  2 files changed, 34 insertions(+), 4 deletions(-)

Acked-by: Stanimir Varbanov <[e-mail]>

Bjorn Helgaas Sept. 20, 2022, 1:37 p.m. UTC | #6

On Tue, Sep 20, 2022 at 10:47:56AM +0200, Johan Hovold wrote:
> Hi Lorenzo,
> 
> On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> > Allow the Qualcomm PCIe controller driver to be built as a module, which
> > is useful for multi-platform kernels as well as during development.
> > 
> > Reviewed-by: Rob Herring <[e-mail]>
> > Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> > Signed-off-by: Johan Hovold <[e-mail]>
> > ---
> > 
> > Changes in v2
> >  - rebase on next-20220720 (adjust context)
> >  - add Rob and Mani's reviewed-by tags
> 
> Have you had a change to look at this one since you got back from
> vacation?
> 
> I believe this should be uncontroversial as we already have other
> modular dwc drivers and there's no mapping of legacy INTx interrupts
> involved.

I'm not Lorenzo, but was there a conclusive outcome to the thread at
[1]?  The last thing I remember was that a buggy endpoint driver that
failed to unmap all its interrupts could cause crashes if the PCIe
controller driver was removed.

Making the driver modular is essential so distros can build all the
drivers and users can load the one needed by their platform.

Making the driver removable is useful for developers but not for
users, so I don't see it as essential.  Developers are in the business
of developing and can easily carry a trivial out-of-tree patch to add
removability if needed.

If removability is actually safe even if endpoint drivers aren't
perfect, then I don't object to it.  But if it's not always safe, I
don't think the argument that "other drivers do it" is strong.  I'd
rather make all the drivers safe even if that means making them
non-removable.

[1] https://lore.kernel.org/r/20220721195433.GA1747571@bhelgaas

> >  drivers/pci/controller/dwc/Kconfig     |  2 +-
> >  drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
> >  2 files changed, 34 insertions(+), 4 deletions(-)
> > 
> > diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
> > index 62ce3abf0f19..230f56d1a268 100644
> > --- a/drivers/pci/controller/dwc/Kconfig
> > +++ b/drivers/pci/controller/dwc/Kconfig
> > @@ -168,7 +168,7 @@ config PCI_HISI
> >  	  Hip05 and Hip06 SoCs
> >  
> >  config PCIE_QCOM
> > -	bool "Qualcomm PCIe controller"
> > +	tristate "Qualcomm PCIe controller"
> >  	depends on OF && (ARCH_QCOM || COMPILE_TEST)
> >  	depends on PCI_MSI_IRQ_DOMAIN
> >  	select PCIE_DW_HOST
> > diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> > index 5ed164c2afa3..d176c635016b 100644
> > --- a/drivers/pci/controller/dwc/pcie-qcom.c
> > +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> > @@ -16,7 +16,7 @@
> >  #include <linux/io.h>
> >  #include <linux/iopoll.h>
> >  #include <linux/kernel.h>
> > -#include <linux/init.h>
> > +#include <linux/module.h>
> >  #include <linux/of_device.h>
> >  #include <linux/of_gpio.h>
> >  #include <linux/pci.h>
> > @@ -1518,6 +1518,15 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
> >  	return ret;
> >  }
> >  
> > +static void qcom_pcie_host_deinit(struct qcom_pcie *pcie)
> > +{
> > +	qcom_ep_reset_assert(pcie);
> > +	if (pcie->cfg->ops->post_deinit)
> > +		pcie->cfg->ops->post_deinit(pcie);
> > +	phy_power_off(pcie->phy);
> > +	pcie->cfg->ops->deinit(pcie);
> > +}
> > +
> >  static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
> >  	.host_init = qcom_pcie_host_init,
> >  };
> > @@ -1752,6 +1761,22 @@ static int qcom_pcie_probe(struct platform_device *pdev)
> >  	return ret;
> >  }
> >  
> > +static int qcom_pcie_remove(struct platform_device *pdev)
> > +{
> > +	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
> > +	struct device *dev = &pdev->dev;
> > +
> > +	dw_pcie_host_deinit(&pcie->pci->pp);
> > +	qcom_pcie_host_deinit(pcie);
> > +
> > +	phy_exit(pcie->phy);
> > +
> > +	pm_runtime_put_sync(dev);
> > +	pm_runtime_disable(dev);
> > +
> > +	return 0;
> > +}
> > +
> >  static const struct of_device_id qcom_pcie_match[] = {
> >  	{ .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
> >  	{ .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
> > @@ -1771,6 +1796,7 @@ static const struct of_device_id qcom_pcie_match[] = {
> >  	{ .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
> >  	{ }
> >  };
> > +MODULE_DEVICE_TABLE(of, qcom_pcie_match);
> >  
> >  static void qcom_fixup_class(struct pci_dev *dev)
> >  {
> > @@ -1786,10 +1812,14 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
> >  
> >  static struct platform_driver qcom_pcie_driver = {
> >  	.probe = qcom_pcie_probe,
> > +	.remove = qcom_pcie_remove,
> >  	.driver = {
> >  		.name = "qcom-pcie",
> > -		.suppress_bind_attrs = true,
> >  		.of_match_table = qcom_pcie_match,
> >  	},
> >  };
> > -builtin_platform_driver(qcom_pcie_driver);
> > +module_platform_driver(qcom_pcie_driver);
> > +
> > +MODULE_AUTHOR("Stanimir Varbanov <[e-mail]>");
> > +MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
> > +MODULE_LICENSE("GPL");
> 
> Johan

Johan Hovold Sept. 20, 2022, 2:05 p.m. UTC | #7

On Tue, Sep 20, 2022 at 08:37:54AM -0500, Bjorn Helgaas wrote:
> On Tue, Sep 20, 2022 at 10:47:56AM +0200, Johan Hovold wrote:
> > Hi Lorenzo,
> > 
> > On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> > > Allow the Qualcomm PCIe controller driver to be built as a module, which
> > > is useful for multi-platform kernels as well as during development.
> > > 
> > > Reviewed-by: Rob Herring <[e-mail]>
> > > Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> > > Signed-off-by: Johan Hovold <[e-mail]>
> > > ---
> > > 
> > > Changes in v2
> > >  - rebase on next-20220720 (adjust context)
> > >  - add Rob and Mani's reviewed-by tags
> > 
> > Have you had a change to look at this one since you got back from
> > vacation?
> > 
> > I believe this should be uncontroversial as we already have other
> > modular dwc drivers and there's no mapping of legacy INTx interrupts
> > involved.
> 
> I'm not Lorenzo, but was there a conclusive outcome to the thread at
> [1]?  The last thing I remember was that a buggy endpoint driver that
> failed to unmap all its interrupts could cause crashes if the PCIe
> controller driver was removed.

That's not so much an argument against allowing the PCIe controller
driver to be unbound as it is an argument for preventing endpoint
drivers from being unbound.

And they generally need to be able to unbind due to hotplugging, right?

> Making the driver modular is essential so distros can build all the
> drivers and users can load the one needed by their platform.
> 
> Making the driver removable is useful for developers but not for
> users, so I don't see it as essential.  Developers are in the business
> of developing and can easily carry a trivial out-of-tree patch to add
> removability if needed.

Having modular drivers that can be unloaded is a debugging feature. I
believe I already posted this quote from Linus:

	The proper thing to do (and what we _have_ done) is to say
	"unloading of modules is not supported". It's a debugging
	feature, and you literally shouldn't do it unless you are
	actively developing that module.

	https://lore.kernel.org/all/[e-mail]/

And no, keeping such patches out-of-tree is not an option as it prevents
sharing them with others and they will quickly bit rot.

> If removability is actually safe even if endpoint drivers aren't
> perfect, then I don't object to it.  But if it's not always safe, I
> don't think the argument that "other drivers do it" is strong.  I'd
> rather make all the drivers safe even if that means making them
> non-removable.

If we have buggy endpoint drivers we need fix them regardless. Allowing
this fundamental debugging feature will even allow developers to catch
those bugs sooner.

And this is really no different from any other type of bug in endpoint
drivers; a failure to deregister a class device on unbind would lead to
crashes due to use-after-free, etc.

> [1] https://lore.kernel.org/r/20220721195433.GA1747571@bhelgaas

Johan

Manivannan Sadhasivam Sept. 20, 2022, 2:27 p.m. UTC | #8

On Tue, Sep 20, 2022 at 08:37:54AM -0500, Bjorn Helgaas wrote:
> On Tue, Sep 20, 2022 at 10:47:56AM +0200, Johan Hovold wrote:
> > Hi Lorenzo,
> > 
> > On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> > > Allow the Qualcomm PCIe controller driver to be built as a module, which
> > > is useful for multi-platform kernels as well as during development.
> > > 
> > > Reviewed-by: Rob Herring <[e-mail]>
> > > Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> > > Signed-off-by: Johan Hovold <[e-mail]>
> > > ---
> > > 
> > > Changes in v2
> > >  - rebase on next-20220720 (adjust context)
> > >  - add Rob and Mani's reviewed-by tags
> > 
> > Have you had a change to look at this one since you got back from
> > vacation?
> > 
> > I believe this should be uncontroversial as we already have other
> > modular dwc drivers and there's no mapping of legacy INTx interrupts
> > involved.
> 
> I'm not Lorenzo, but was there a conclusive outcome to the thread at
> [1]?  The last thing I remember was that a buggy endpoint driver that
> failed to unmap all its interrupts could cause crashes if the PCIe
> controller driver was removed.
> 
> Making the driver modular is essential so distros can build all the
> drivers and users can load the one needed by their platform.
> 
> Making the driver removable is useful for developers but not for
> users, so I don't see it as essential.  Developers are in the business
> of developing and can easily carry a trivial out-of-tree patch to add
> removability if needed.
> 
> If removability is actually safe even if endpoint drivers aren't
> perfect, then I don't object to it.  But if it's not always safe, I
> don't think the argument that "other drivers do it" is strong.  I'd
> rather make all the drivers safe even if that means making them
> non-removable.
> 

If the problem is with the endpoint drivers, then those need to be fixed for
properly disposing the IRQ mappings. I don't see any real issue with the
remove() handler of this controller driver.

Thanks,
Mani

> [1] https://lore.kernel.org/r/20220721195433.GA1747571@bhelgaas
> 
> > >  drivers/pci/controller/dwc/Kconfig     |  2 +-
> > >  drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
> > >  2 files changed, 34 insertions(+), 4 deletions(-)
> > > 
> > > diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
> > > index 62ce3abf0f19..230f56d1a268 100644
> > > --- a/drivers/pci/controller/dwc/Kconfig
> > > +++ b/drivers/pci/controller/dwc/Kconfig
> > > @@ -168,7 +168,7 @@ config PCI_HISI
> > >  	  Hip05 and Hip06 SoCs
> > >  
> > >  config PCIE_QCOM
> > > -	bool "Qualcomm PCIe controller"
> > > +	tristate "Qualcomm PCIe controller"
> > >  	depends on OF && (ARCH_QCOM || COMPILE_TEST)
> > >  	depends on PCI_MSI_IRQ_DOMAIN
> > >  	select PCIE_DW_HOST
> > > diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> > > index 5ed164c2afa3..d176c635016b 100644
> > > --- a/drivers/pci/controller/dwc/pcie-qcom.c
> > > +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> > > @@ -16,7 +16,7 @@
> > >  #include <linux/io.h>
> > >  #include <linux/iopoll.h>
> > >  #include <linux/kernel.h>
> > > -#include <linux/init.h>
> > > +#include <linux/module.h>
> > >  #include <linux/of_device.h>
> > >  #include <linux/of_gpio.h>
> > >  #include <linux/pci.h>
> > > @@ -1518,6 +1518,15 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
> > >  	return ret;
> > >  }
> > >  
> > > +static void qcom_pcie_host_deinit(struct qcom_pcie *pcie)
> > > +{
> > > +	qcom_ep_reset_assert(pcie);
> > > +	if (pcie->cfg->ops->post_deinit)
> > > +		pcie->cfg->ops->post_deinit(pcie);
> > > +	phy_power_off(pcie->phy);
> > > +	pcie->cfg->ops->deinit(pcie);
> > > +}
> > > +
> > >  static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
> > >  	.host_init = qcom_pcie_host_init,
> > >  };
> > > @@ -1752,6 +1761,22 @@ static int qcom_pcie_probe(struct platform_device *pdev)
> > >  	return ret;
> > >  }
> > >  
> > > +static int qcom_pcie_remove(struct platform_device *pdev)
> > > +{
> > > +	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
> > > +	struct device *dev = &pdev->dev;
> > > +
> > > +	dw_pcie_host_deinit(&pcie->pci->pp);
> > > +	qcom_pcie_host_deinit(pcie);
> > > +
> > > +	phy_exit(pcie->phy);
> > > +
> > > +	pm_runtime_put_sync(dev);
> > > +	pm_runtime_disable(dev);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > >  static const struct of_device_id qcom_pcie_match[] = {
> > >  	{ .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
> > >  	{ .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
> > > @@ -1771,6 +1796,7 @@ static const struct of_device_id qcom_pcie_match[] = {
> > >  	{ .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
> > >  	{ }
> > >  };
> > > +MODULE_DEVICE_TABLE(of, qcom_pcie_match);
> > >  
> > >  static void qcom_fixup_class(struct pci_dev *dev)
> > >  {
> > > @@ -1786,10 +1812,14 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
> > >  
> > >  static struct platform_driver qcom_pcie_driver = {
> > >  	.probe = qcom_pcie_probe,
> > > +	.remove = qcom_pcie_remove,
> > >  	.driver = {
> > >  		.name = "qcom-pcie",
> > > -		.suppress_bind_attrs = true,
> > >  		.of_match_table = qcom_pcie_match,
> > >  	},
> > >  };
> > > -builtin_platform_driver(qcom_pcie_driver);
> > > +module_platform_driver(qcom_pcie_driver);
> > > +
> > > +MODULE_AUTHOR("Stanimir Varbanov <[e-mail]>");
> > > +MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
> > > +MODULE_LICENSE("GPL");
> > 
> > Johan

Lorenzo Pieralisi Sept. 27, 2022, 1:28 p.m. UTC | #9

On Tue, Sep 20, 2022 at 10:47:56AM +0200, Johan Hovold wrote:
> Hi Lorenzo,
> 
> On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> > Allow the Qualcomm PCIe controller driver to be built as a module, which
> > is useful for multi-platform kernels as well as during development.
> > 
> > Reviewed-by: Rob Herring <[e-mail]>
> > Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> > Signed-off-by: Johan Hovold <[e-mail]>
> > ---
> > 
> > Changes in v2
> >  - rebase on next-20220720 (adjust context)
> >  - add Rob and Mani's reviewed-by tags
> 
> Have you had a change to look at this one since you got back from
> vacation?
> 
> I believe this should be uncontroversial as we already have other
> modular dwc drivers and there's no mapping of legacy INTx interrupts
> involved.

Sincere apologies for the delay.

I am afraid it does look controversial - I need some time to go through
the full discussion and make up my mind, unfortunately we are late in
the cycle and I am dealing with the patch backlog, I believe this may
end up being a discussion targeting the v6.2 merge window I am afraid.

Lorenzo

> >  drivers/pci/controller/dwc/Kconfig     |  2 +-
> >  drivers/pci/controller/dwc/pcie-qcom.c | 36 +++++++++++++++++++++++---
> >  2 files changed, 34 insertions(+), 4 deletions(-)
> > 
> > diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
> > index 62ce3abf0f19..230f56d1a268 100644
> > --- a/drivers/pci/controller/dwc/Kconfig
> > +++ b/drivers/pci/controller/dwc/Kconfig
> > @@ -168,7 +168,7 @@ config PCI_HISI
> >  	  Hip05 and Hip06 SoCs
> >  
> >  config PCIE_QCOM
> > -	bool "Qualcomm PCIe controller"
> > +	tristate "Qualcomm PCIe controller"
> >  	depends on OF && (ARCH_QCOM || COMPILE_TEST)
> >  	depends on PCI_MSI_IRQ_DOMAIN
> >  	select PCIE_DW_HOST
> > diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> > index 5ed164c2afa3..d176c635016b 100644
> > --- a/drivers/pci/controller/dwc/pcie-qcom.c
> > +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> > @@ -16,7 +16,7 @@
> >  #include <linux/io.h>
> >  #include <linux/iopoll.h>
> >  #include <linux/kernel.h>
> > -#include <linux/init.h>
> > +#include <linux/module.h>
> >  #include <linux/of_device.h>
> >  #include <linux/of_gpio.h>
> >  #include <linux/pci.h>
> > @@ -1518,6 +1518,15 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
> >  	return ret;
> >  }
> >  
> > +static void qcom_pcie_host_deinit(struct qcom_pcie *pcie)
> > +{
> > +	qcom_ep_reset_assert(pcie);
> > +	if (pcie->cfg->ops->post_deinit)
> > +		pcie->cfg->ops->post_deinit(pcie);
> > +	phy_power_off(pcie->phy);
> > +	pcie->cfg->ops->deinit(pcie);
> > +}
> > +
> >  static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
> >  	.host_init = qcom_pcie_host_init,
> >  };
> > @@ -1752,6 +1761,22 @@ static int qcom_pcie_probe(struct platform_device *pdev)
> >  	return ret;
> >  }
> >  
> > +static int qcom_pcie_remove(struct platform_device *pdev)
> > +{
> > +	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
> > +	struct device *dev = &pdev->dev;
> > +
> > +	dw_pcie_host_deinit(&pcie->pci->pp);
> > +	qcom_pcie_host_deinit(pcie);
> > +
> > +	phy_exit(pcie->phy);
> > +
> > +	pm_runtime_put_sync(dev);
> > +	pm_runtime_disable(dev);
> > +
> > +	return 0;
> > +}
> > +
> >  static const struct of_device_id qcom_pcie_match[] = {
> >  	{ .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
> >  	{ .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
> > @@ -1771,6 +1796,7 @@ static const struct of_device_id qcom_pcie_match[] = {
> >  	{ .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
> >  	{ }
> >  };
> > +MODULE_DEVICE_TABLE(of, qcom_pcie_match);
> >  
> >  static void qcom_fixup_class(struct pci_dev *dev)
> >  {
> > @@ -1786,10 +1812,14 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
> >  
> >  static struct platform_driver qcom_pcie_driver = {
> >  	.probe = qcom_pcie_probe,
> > +	.remove = qcom_pcie_remove,
> >  	.driver = {
> >  		.name = "qcom-pcie",
> > -		.suppress_bind_attrs = true,
> >  		.of_match_table = qcom_pcie_match,
> >  	},
> >  };
> > -builtin_platform_driver(qcom_pcie_driver);
> > +module_platform_driver(qcom_pcie_driver);
> > +
> > +MODULE_AUTHOR("Stanimir Varbanov <[e-mail]>");
> > +MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
> > +MODULE_LICENSE("GPL");
> 
> Johan

Johan Hovold Sept. 28, 2022, 6:16 a.m. UTC | #10

On Tue, Sep 27, 2022 at 03:28:09PM +0200, Lorenzo Pieralisi wrote:
> On Tue, Sep 20, 2022 at 10:47:56AM +0200, Johan Hovold wrote:
> > Hi Lorenzo,
> > 
> > On Thu, Jul 21, 2022 at 08:47:20AM +0200, Johan Hovold wrote:
> > > Allow the Qualcomm PCIe controller driver to be built as a module, which
> > > is useful for multi-platform kernels as well as during development.
> > > 
> > > Reviewed-by: Rob Herring <[e-mail]>
> > > Reviewed-by: Manivannan Sadhasivam <[e-mail]>
> > > Signed-off-by: Johan Hovold <[e-mail]>
> > > ---
> > > 
> > > Changes in v2
> > >  - rebase on next-20220720 (adjust context)
> > >  - add Rob and Mani's reviewed-by tags
> > 
> > Have you had a change to look at this one since you got back from
> > vacation?
> > 
> > I believe this should be uncontroversial as we already have other
> > modular dwc drivers and there's no mapping of legacy INTx interrupts
> > involved.
> 
> Sincere apologies for the delay.
> 
> I am afraid it does look controversial - I need some time to go through
> the full discussion and make up my mind, unfortunately we are late in
> the cycle and I am dealing with the patch backlog, I believe this may
> end up being a discussion targeting the v6.2 merge window I am afraid.

No worries. Thanks for taking a look.

Johan

Patch

diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index 62ce3abf0f19..230f56d1a268 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -168,7 +168,7 @@  config PCI_HISI
 	  Hip05 and Hip06 SoCs
 
 config PCIE_QCOM
-	bool "Qualcomm PCIe controller"
+	tristate "Qualcomm PCIe controller"
 	depends on OF && (ARCH_QCOM || COMPILE_TEST)
 	depends on PCI_MSI_IRQ_DOMAIN
 	select PCIE_DW_HOST
diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
index 5ed164c2afa3..d176c635016b 100644
--- a/drivers/pci/controller/dwc/pcie-qcom.c
+++ b/drivers/pci/controller/dwc/pcie-qcom.c
@@ -16,7 +16,7 @@ 
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/kernel.h>
-#include <linux/init.h>
+#include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
 #include <linux/pci.h>
@@ -1518,6 +1518,15 @@  static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
 	return ret;
 }
 
+static void qcom_pcie_host_deinit(struct qcom_pcie *pcie)
+{
+	qcom_ep_reset_assert(pcie);
+	if (pcie->cfg->ops->post_deinit)
+		pcie->cfg->ops->post_deinit(pcie);
+	phy_power_off(pcie->phy);
+	pcie->cfg->ops->deinit(pcie);
+}
+
 static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
 	.host_init = qcom_pcie_host_init,
 };
@@ -1752,6 +1761,22 @@  static int qcom_pcie_probe(struct platform_device *pdev)
 	return ret;
 }
 
+static int qcom_pcie_remove(struct platform_device *pdev)
+{
+	struct qcom_pcie *pcie = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+
+	dw_pcie_host_deinit(&pcie->pci->pp);
+	qcom_pcie_host_deinit(pcie);
+
+	phy_exit(pcie->phy);
+
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+
+	return 0;
+}
+
 static const struct of_device_id qcom_pcie_match[] = {
 	{ .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg },
 	{ .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg },
@@ -1771,6 +1796,7 @@  static const struct of_device_id qcom_pcie_match[] = {
 	{ .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, qcom_pcie_match);
 
 static void qcom_fixup_class(struct pci_dev *dev)
 {
@@ -1786,10 +1812,14 @@  DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class);
 
 static struct platform_driver qcom_pcie_driver = {
 	.probe = qcom_pcie_probe,
+	.remove = qcom_pcie_remove,
 	.driver = {
 		.name = "qcom-pcie",
-		.suppress_bind_attrs = true,
 		.of_match_table = qcom_pcie_match,
 	},
 };
-builtin_platform_driver(qcom_pcie_driver);
+module_platform_driver(qcom_pcie_driver);
+
+MODULE_AUTHOR("Stanimir Varbanov <[e-mail]>");
+MODULE_DESCRIPTION("Qualcomm PCIe root complex driver");
+MODULE_LICENSE("GPL");