[v18,1/5] iommu/arm-smmu: Add pm_runtime/sleep ops

Commit Message

Vivek Gautam Nov. 27, 2018, 10:11 a.m. UTC

From: Sricharan R <[e-mail]>

The smmu needs to be functional only when the respective
master's using it are active. The device_link feature
helps to track such functional dependencies, so that the
iommu gets powered when the master device enables itself
using pm_runtime. So by adapting the smmu driver for
runtime pm, above said dependency can be addressed.

This patch adds the pm runtime/sleep callbacks to the
driver and also the functions to parse the smmu clocks
from DT and enable them in resume/suspend.
We pull all the information about clocks from device tree.

Also, while we enable the runtime pm add a pm sleep suspend
callback that pushes devices to low power state by turning
the clocks off in a system sleep.
Also add corresponding clock enable path in resume callback.

Signed-off-by: Sricharan R <[e-mail]>
Signed-off-by: Archit Taneja <[e-mail]>
[Thor: Rework to get clocks from device tree]
Signed-off-by: Thor Thayer <[e-mail]>
[vivek: rework for clock and pm ops]
Signed-off-by: Vivek Gautam <[e-mail]>
Reviewed-by: Tomasz Figa <[e-mail]>
Tested-by: Srinivas Kandagatla <[e-mail]>
Reviewed-by: Robin Murphy <[e-mail]>
---
 drivers/iommu/arm-smmu.c | 100 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 97 insertions(+), 3 deletions(-)

Comments

Thor Thayer Nov. 28, 2018, 2:32 p.m. UTC | #1

On 11/27/18 4:11 AM, Vivek Gautam wrote:
> From: Sricharan R <[e-mail]>
> 
> The smmu needs to be functional only when the respective
> master's using it are active. The device_link feature
> helps to track such functional dependencies, so that the
> iommu gets powered when the master device enables itself
> using pm_runtime. So by adapting the smmu driver for
> runtime pm, above said dependency can be addressed.
> 
> This patch adds the pm runtime/sleep callbacks to the
> driver and also the functions to parse the smmu clocks
> from DT and enable them in resume/suspend.
> We pull all the information about clocks from device tree.
> 
> Also, while we enable the runtime pm add a pm sleep suspend
> callback that pushes devices to low power state by turning
> the clocks off in a system sleep.
> Also add corresponding clock enable path in resume callback.
> 
> Signed-off-by: Sricharan R <[e-mail]>
> Signed-off-by: Archit Taneja <[e-mail]>
> [Thor: Rework to get clocks from device tree]
> Signed-off-by: Thor Thayer <[e-mail]>
> [vivek: rework for clock and pm ops]
> Signed-off-by: Vivek Gautam <[e-mail]>
> Reviewed-by: Tomasz Figa <[e-mail]>
> Tested-by: Srinivas Kandagatla <[e-mail]>
> Reviewed-by: Robin Murphy <[e-mail]>
> ---
>   drivers/iommu/arm-smmu.c | 100 +++++++++++++++++++++++++++++++++++++++++++++--
>   1 file changed, 97 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
> index 5a28ae892504..e47c840fc6a8 100644
> --- a/drivers/iommu/arm-smmu.c
> +++ b/drivers/iommu/arm-smmu.c
> @@ -44,10 +44,12 @@
>   #include <linux/module.h>
>   #include <linux/of.h>
>   #include <linux/of_address.h>
> +#include <linux/of_clk.h>
>   #include <linux/of_device.h>
>   #include <linux/of_iommu.h>
>   #include <linux/pci.h>
>   #include <linux/platform_device.h>
> +#include <linux/pm_runtime.h>
>   #include <linux/slab.h>
>   #include <linux/spinlock.h>
>  <snip>

Thanks! Tested the device tree clock portions on Intel SOCFPGA Stratix10 
DevKit.

Tested-by: Thor Thayer <[e-mail]>

Stephen Boyd Nov. 28, 2018, 4:24 p.m. UTC | #2

Quoting Vivek Gautam (2018-11-27 02:11:41)
> @@ -1966,6 +1970,23 @@ static const struct of_device_id arm_smmu_of_match[] = {
>  };
>  MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
>  
> +static void arm_smmu_fill_clk_data(struct arm_smmu_device *smmu,
> +                                  const char * const *clks)
> +{
> +       int i;
> +
> +       if (smmu->num_clks < 1)
> +               return;
> +
> +       smmu->clks = devm_kcalloc(smmu->dev, smmu->num_clks,
> +                                 sizeof(*smmu->clks), GFP_KERNEL);
> +       if (!smmu->clks)
> +               return;
> +
> +       for (i = 0; i < smmu->num_clks; i++)
> +               smmu->clks[i].id = clks[i];

Is this clk_bulk_get_all()?

Robin Murphy Nov. 28, 2018, 4:36 p.m. UTC | #3

On 28/11/2018 16:24, Stephen Boyd wrote:
> Quoting Vivek Gautam (2018-11-27 02:11:41)
>> @@ -1966,6 +1970,23 @@ static const struct of_device_id arm_smmu_of_match[] = {
>>   };
>>   MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
>>   
>> +static void arm_smmu_fill_clk_data(struct arm_smmu_device *smmu,
>> +                                  const char * const *clks)
>> +{
>> +       int i;
>> +
>> +       if (smmu->num_clks < 1)
>> +               return;
>> +
>> +       smmu->clks = devm_kcalloc(smmu->dev, smmu->num_clks,
>> +                                 sizeof(*smmu->clks), GFP_KERNEL);
>> +       if (!smmu->clks)
>> +               return;
>> +
>> +       for (i = 0; i < smmu->num_clks; i++)
>> +               smmu->clks[i].id = clks[i];
> 
> Is this clk_bulk_get_all()?

Ooh, did that finally get merged while we weren't looking? Great!

Much as I don't want to drag this series out to a v19, it *would* be 
neat if we no longer need to open-code that bit...

Robin.

Vivek Gautam Nov. 29, 2018, 2:55 p.m. UTC | #4

On Wed, Nov 28, 2018 at 10:07 PM Robin Murphy <[e-mail]> wrote:
>
> On 28/11/2018 16:24, Stephen Boyd wrote:
> > Quoting Vivek Gautam (2018-11-27 02:11:41)
> >> @@ -1966,6 +1970,23 @@ static const struct of_device_id arm_smmu_of_match[] = {
> >>   };
> >>   MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
> >>
> >> +static void arm_smmu_fill_clk_data(struct arm_smmu_device *smmu,
> >> +                                  const char * const *clks)
> >> +{
> >> +       int i;
> >> +
> >> +       if (smmu->num_clks < 1)
> >> +               return;
> >> +
> >> +       smmu->clks = devm_kcalloc(smmu->dev, smmu->num_clks,
> >> +                                 sizeof(*smmu->clks), GFP_KERNEL);
> >> +       if (!smmu->clks)
> >> +               return;
> >> +
> >> +       for (i = 0; i < smmu->num_clks; i++)
> >> +               smmu->clks[i].id = clks[i];
> >
> > Is this clk_bulk_get_all()?

From what I remember, and now I could go back to v7 and check [1], we parked
clk_bulk_get out of OF's sole purview as we also have
arm_smmu_device_acpi_probe() besides arm_smmu_device_dt_probe().

arm_smmu_device_dt_probe() could get the clocks from dt and fill in
the clock bulk data, and
similarly, arm_smmu_device_acpi_probe() could fill the clock bulk data
by getting it from ACPI.

clk_bulk_get_all() seems like going only the OF way.
Is there another way here to have something common between ACPI
and OF, and then do the clk_bulk_get?

[1] https://lore.kernel.org/patchwork/patch/881365/

Thanks & regards
Vivek

>
> Ooh, did that finally get merged while we weren't looking? Great!
>
> Much as I don't want to drag this series out to a v19, it *would* be
> neat if we no longer need to open-code that bit...
>
> Robin.
> _______________________________________________
> iommu mailing list
> [e-mail]
> https://lists.linuxfoundation.org/mailman/listinfo/iommu



--
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member
of Code Aurora Forum, hosted by The Linux Foundation

Will Deacon Nov. 30, 2018, 6:15 p.m. UTC | #5

On Thu, Nov 29, 2018 at 08:25:20PM +0530, Vivek Gautam wrote:
> On Wed, Nov 28, 2018 at 10:07 PM Robin Murphy <[e-mail]> wrote:
> >
> > On 28/11/2018 16:24, Stephen Boyd wrote:
> > > Quoting Vivek Gautam (2018-11-27 02:11:41)
> > >> @@ -1966,6 +1970,23 @@ static const struct of_device_id arm_smmu_of_match[] = {
> > >>   };
> > >>   MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
> > >>
> > >> +static void arm_smmu_fill_clk_data(struct arm_smmu_device *smmu,
> > >> +                                  const char * const *clks)
> > >> +{
> > >> +       int i;
> > >> +
> > >> +       if (smmu->num_clks < 1)
> > >> +               return;
> > >> +
> > >> +       smmu->clks = devm_kcalloc(smmu->dev, smmu->num_clks,
> > >> +                                 sizeof(*smmu->clks), GFP_KERNEL);
> > >> +       if (!smmu->clks)
> > >> +               return;
> > >> +
> > >> +       for (i = 0; i < smmu->num_clks; i++)
> > >> +               smmu->clks[i].id = clks[i];
> > >
> > > Is this clk_bulk_get_all()?
> 
> From what I remember, and now I could go back to v7 and check [1], we parked
> clk_bulk_get out of OF's sole purview as we also have
> arm_smmu_device_acpi_probe() besides arm_smmu_device_dt_probe().
> 
> arm_smmu_device_dt_probe() could get the clocks from dt and fill in
> the clock bulk data, and
> similarly, arm_smmu_device_acpi_probe() could fill the clock bulk data
> by getting it from ACPI.
> 
> clk_bulk_get_all() seems like going only the OF way.
> Is there another way here to have something common between ACPI
> and OF, and then do the clk_bulk_get?

I'd say just go with clk_bulk_get_all() and if somebody really wants to
mess with the SMMU clocks on a system booted via ACPI, then it's their
problem to solve. My understanding is that the design of IORT makes this
next to impossible to solve anyway, because a static table is used and
therefore we're unable to run whatever ASL methods need to be invoked to
mess with the clocks.

Will

Vivek Gautam Dec. 3, 2018, 6:43 a.m. UTC | #6

On Fri, Nov 30, 2018 at 11:45 PM Will Deacon <[e-mail]> wrote:
>
> On Thu, Nov 29, 2018 at 08:25:20PM +0530, Vivek Gautam wrote:
> > On Wed, Nov 28, 2018 at 10:07 PM Robin Murphy <[e-mail]> wrote:
> > >
> > > On 28/11/2018 16:24, Stephen Boyd wrote:
> > > > Quoting Vivek Gautam (2018-11-27 02:11:41)
> > > >> @@ -1966,6 +1970,23 @@ static const struct of_device_id arm_smmu_of_match[] = {
> > > >>   };
> > > >>   MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
> > > >>
> > > >> +static void arm_smmu_fill_clk_data(struct arm_smmu_device *smmu,
> > > >> +                                  const char * const *clks)
> > > >> +{
> > > >> +       int i;
> > > >> +
> > > >> +       if (smmu->num_clks < 1)
> > > >> +               return;
> > > >> +
> > > >> +       smmu->clks = devm_kcalloc(smmu->dev, smmu->num_clks,
> > > >> +                                 sizeof(*smmu->clks), GFP_KERNEL);
> > > >> +       if (!smmu->clks)
> > > >> +               return;
> > > >> +
> > > >> +       for (i = 0; i < smmu->num_clks; i++)
> > > >> +               smmu->clks[i].id = clks[i];
> > > >
> > > > Is this clk_bulk_get_all()?
> >
> > From what I remember, and now I could go back to v7 and check [1], we parked
> > clk_bulk_get out of OF's sole purview as we also have
> > arm_smmu_device_acpi_probe() besides arm_smmu_device_dt_probe().
> >
> > arm_smmu_device_dt_probe() could get the clocks from dt and fill in
> > the clock bulk data, and
> > similarly, arm_smmu_device_acpi_probe() could fill the clock bulk data
> > by getting it from ACPI.
> >
> > clk_bulk_get_all() seems like going only the OF way.
> > Is there another way here to have something common between ACPI
> > and OF, and then do the clk_bulk_get?
>
> I'd say just go with clk_bulk_get_all() and if somebody really wants to
> mess with the SMMU clocks on a system booted via ACPI, then it's their
> problem to solve. My understanding is that the design of IORT makes this
> next to impossible to solve anyway, because a static table is used and
> therefore we're unable to run whatever ASL methods need to be invoked to
> mess with the clocks.

Sure then. I will respin this patch-series.

>
> Will

Stephen Boyd Dec. 3, 2018, 11:44 p.m. UTC | #7

Quoting Vivek Gautam (2018-12-02 22:43:38)
> On Fri, Nov 30, 2018 at 11:45 PM Will Deacon <[e-mail]> wrote:
> >
> > On Thu, Nov 29, 2018 at 08:25:20PM +0530, Vivek Gautam wrote:
> > > clk_bulk_get_all() seems like going only the OF way.
> > > Is there another way here to have something common between ACPI
> > > and OF, and then do the clk_bulk_get?
> >
> > I'd say just go with clk_bulk_get_all() and if somebody really wants to
> > mess with the SMMU clocks on a system booted via ACPI, then it's their
> > problem to solve. My understanding is that the design of IORT makes this
> > next to impossible to solve anyway, because a static table is used and
> > therefore we're unable to run whatever ASL methods need to be invoked to
> > mess with the clocks.
> 
> Sure then. I will respin this patch-series.
> 

Right. The idea is to add non-OF support to clk_bulk_get_all() if/when
we get the requirement. Sounds like we can keep waiting a little longer
for that to happen.

Patch

diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 5a28ae892504..e47c840fc6a8 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -44,10 +44,12 @@ 
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_clk.h>
 #include <linux/of_device.h>
 #include <linux/of_iommu.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
@@ -206,6 +208,8 @@  struct arm_smmu_device {
 	u32				num_global_irqs;
 	u32				num_context_irqs;
 	unsigned int			*irqs;
+	struct clk_bulk_data		*clks;
+	int				num_clks;
 
 	u32				cavium_id_base; /* Specific to Cavium */
 
@@ -1947,7 +1951,7 @@  struct arm_smmu_match_data {
 };
 
 #define ARM_SMMU_MATCH_DATA(name, ver, imp)	\
-static struct arm_smmu_match_data name = { .version = ver, .model = imp }
+static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
 
 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
@@ -1966,6 +1970,23 @@  static const struct of_device_id arm_smmu_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
 
+static void arm_smmu_fill_clk_data(struct arm_smmu_device *smmu,
+				   const char * const *clks)
+{
+	int i;
+
+	if (smmu->num_clks < 1)
+		return;
+
+	smmu->clks = devm_kcalloc(smmu->dev, smmu->num_clks,
+				  sizeof(*smmu->clks), GFP_KERNEL);
+	if (!smmu->clks)
+		return;
+
+	for (i = 0; i < smmu->num_clks; i++)
+		smmu->clks[i].id = clks[i];
+}
+
 #ifdef CONFIG_ACPI
 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
 {
@@ -2038,6 +2059,7 @@  static int arm_smmu_device_dt_probe(struct platform_device *pdev,
 	const struct arm_smmu_match_data *data;
 	struct device *dev = &pdev->dev;
 	bool legacy_binding;
+	const char **parent_names;
 
 	if (of_property_read_u32(dev->of_node, "#global-interrupts",
 				 &smmu->num_global_irqs)) {
@@ -2048,6 +2070,26 @@  static int arm_smmu_device_dt_probe(struct platform_device *pdev,
 	data = of_device_get_match_data(dev);
 	smmu->version = data->version;
 	smmu->model = data->model;
+	smmu->num_clks = of_clk_get_parent_count(dev->of_node);
+	/* check to see if clocks were specified in DT */
+	if (smmu->num_clks) {
+		unsigned int i;
+
+		parent_names = kmalloc_array(smmu->num_clks,
+					     sizeof(*parent_names),
+					     GFP_KERNEL);
+		if (!parent_names)
+			return -ENOMEM;
+
+		for (i = 0; i < smmu->num_clks; i++) {
+			if (of_property_read_string_index(dev->of_node,
+							  "clock-names", i,
+							  &parent_names[i]))
+				goto fail_clk_name;
+		}
+		arm_smmu_fill_clk_data(smmu, parent_names);
+		kfree(parent_names);
+	}
 
 	parse_driver_options(smmu);
 
@@ -2067,6 +2109,12 @@  static int arm_smmu_device_dt_probe(struct platform_device *pdev,
 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
 
 	return 0;
+
+fail_clk_name:
+	kfree(parent_names);
+	/* clock-names required for clocks in devm_clk_bulk_get() */
+	dev_err(dev, "clock-names required in device tree\n");
+	return -ENODEV;
 }
 
 static void arm_smmu_bus_init(void)
@@ -2150,6 +2198,14 @@  static int arm_smmu_device_probe(struct platform_device *pdev)
 		smmu->irqs[i] = irq;
 	}
 
+	err = devm_clk_bulk_get(smmu->dev, smmu->num_clks, smmu->clks);
+	if (err)
+		return err;
+
+	err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
+	if (err)
+		return err;
+
 	err = arm_smmu_device_cfg_probe(smmu);
 	if (err)
 		return err;
@@ -2236,6 +2292,9 @@  static int arm_smmu_device_remove(struct platform_device *pdev)
 
 	/* Turn the thing off */
 	writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
+
+	clk_bulk_disable_unprepare(smmu->num_clks, smmu->clks);
+
 	return 0;
 }
 
@@ -2244,15 +2303,50 @@  static void arm_smmu_device_shutdown(struct platform_device *pdev)
 	arm_smmu_device_remove(pdev);
 }
 
-static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
+static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
 {
 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
+	if (ret)
+		return ret;
 
 	arm_smmu_device_reset(smmu);
+
+	return 0;
+}
+
+static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
+{
+	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
+
+	clk_bulk_disable(smmu->num_clks, smmu->clks);
+
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume);
+static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
+{
+	if (pm_runtime_suspended(dev))
+		return 0;
+
+	return arm_smmu_runtime_resume(dev);
+}
+
+static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
+{
+	if (pm_runtime_suspended(dev))
+		return 0;
+
+	return arm_smmu_runtime_suspend(dev);
+}
+
+static const struct dev_pm_ops arm_smmu_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
+	SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
+			   arm_smmu_runtime_resume, NULL)
+};
 
 static struct platform_driver arm_smmu_driver = {
 	.driver	= {