[v9,3/5] iommu/arm-smmu: introduction of ACTLR for custom prefetcher settings

Commit Message

Bibek Kumar Patro Jan. 23, 2024, 2:45 p.m. UTC

Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
the TLB to fetch just the next page table. MMU-500 features ACTLR
register which is implementation defined and is used for Qualcomm SoCs
to have a custom prefetch setting enabling TLB to prefetch the next set
of page tables accordingly allowing for faster translations.

ACTLR value is unique for each SMR (Stream matching register) and stored
in a pre-populated table. This value is set to the register during
context bank initialisation.

Signed-off-by: Bibek Kumar Patro <[e-mail]>
---
 drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 61 ++++++++++++++++++++++
 drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 16 +++++-
 drivers/iommu/arm/arm-smmu/arm-smmu.c      |  5 +-
 drivers/iommu/arm/arm-smmu/arm-smmu.h      |  5 ++
 4 files changed, 84 insertions(+), 3 deletions(-)

--
2.17.1

Comments

Bibek Kumar Patro Feb. 9, 2024, 9:55 a.m. UTC | #1

Hi,

Are there any additional inputs or suggestions on this patch?
Could you let me know please, incase there are any further concerns.
will try to address them.

Thanks & regards,
Bibek

On 1/23/2024 8:15 PM, Bibek Kumar Patro wrote:
> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> the TLB to fetch just the next page table. MMU-500 features ACTLR
> register which is implementation defined and is used for Qualcomm SoCs
> to have a custom prefetch setting enabling TLB to prefetch the next set
> of page tables accordingly allowing for faster translations.
> 
> ACTLR value is unique for each SMR (Stream matching register) and stored
> in a pre-populated table. This value is set to the register during
> context bank initialisation.
> 
> Signed-off-by: Bibek Kumar Patro <[e-mail]>
> ---
>   drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 61 ++++++++++++++++++++++
>   drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 16 +++++-
>   drivers/iommu/arm/arm-smmu/arm-smmu.c      |  5 +-
>   drivers/iommu/arm/arm-smmu/arm-smmu.h      |  5 ++
>   4 files changed, 84 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> index 333daeb18c1c..6004c6d9a7b2 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> @@ -215,10 +215,42 @@ static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
>   	return true;
>   }
> 
> +static void qcom_smmu_set_actlr(struct device *dev, struct arm_smmu_device *smmu, int cbndx,
> +		const struct actlr_config *actlrcfg, const size_t num_actlrcfg)
> +{
> +	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
> +	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
> +	struct arm_smmu_smr *smr;
> +	u16 mask;
> +	int idx;
> +	u16 id;
> +	int i;
> +	int j;
> +
> +	for (i = 0; i < num_actlrcfg; i++) {
> +		id = actlrcfg[i].sid;
> +		mask = actlrcfg[i].mask;
> +
> +		for_each_cfg_sme(cfg, fwspec, j, idx) {
> +			smr = &smmu->smrs[idx];
> +			if (smr_is_subset(smr, id, mask)) {
> +				arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> +						actlrcfg[i].actlr);
> +				break;
> +			}
> +		}
> +	}
> +}
> +
>   static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>   		struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>   {
> +	struct arm_smmu_device *smmu = smmu_domain->smmu;
> +	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
> +	const struct actlr_variant *actlrvar;
> +	int cbndx = smmu_domain->cfg.cbndx;
>   	struct adreno_smmu_priv *priv;
> +	int i;
> 
>   	smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
> 
> @@ -248,6 +280,18 @@ static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>   	priv->set_stall = qcom_adreno_smmu_set_stall;
>   	priv->resume_translation = qcom_adreno_smmu_resume_translation;
> 
> +	actlrvar = qsmmu->data->actlrvar;
> +	if (!actlrvar)
> +		return 0;
> +
> +	for (i = 0; i < qsmmu->data->num_smmu ; i++) {
> +		if (actlrvar[i].io_start == smmu->ioaddr) {
> +			qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
> +				       actlrvar[i].num_actlrcfg);
> +			break;
> +		}
> +	}
> +
>   	return 0;
>   }
> 
> @@ -274,7 +318,24 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
>   static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>   		struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>   {
> +	struct arm_smmu_device *smmu = smmu_domain->smmu;
> +	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
> +	const struct actlr_variant *actlrvar;
> +	int cbndx = smmu_domain->cfg.cbndx;
> +	int i;
> +
>   	smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
> +	actlrvar = qsmmu->data->actlrvar;
> +	if (!actlrvar)
> +		return 0;
> +
> +	for (i = 0; i < qsmmu->data->num_smmu ; i++) {
> +		if (actlrvar[i].io_start == smmu->ioaddr) {
> +			qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
> +				       actlrvar[i].num_actlrcfg);
> +			break;
> +		}
> +	}
> 
>   	return 0;
>   }
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> index f3b91963e234..3f651242de7c 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> @@ -1,6 +1,6 @@
>   /* SPDX-License-Identifier: GPL-2.0-only */
>   /*
> - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
> + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
>    */
> 
>   #ifndef _ARM_SMMU_QCOM_H
> @@ -24,8 +24,22 @@ struct qcom_smmu_config {
>   	const u32 *reg_offset;
>   };
> 
> +struct actlr_config {
> +	u16 sid;
> +	u16 mask;
> +	u32 actlr;
> +};
> +
> +struct actlr_variant {
> +	const resource_size_t io_start;
> +	const struct actlr_config * const actlrcfg;
> +	const size_t num_actlrcfg;
> +};
> +
>   struct qcom_smmu_match_data {
> +	const struct actlr_variant * const actlrvar;
>   	const struct qcom_smmu_config *cfg;
> +	const size_t num_smmu;
>   	const struct arm_smmu_impl *impl;
>   	const struct arm_smmu_impl *adreno_impl;
>   };
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c
> index d6d1a2a55cc0..0c7f700b27dd 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
> @@ -990,9 +990,10 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
>   		 * expect simply identical entries for this case, but there's
>   		 * no harm in accommodating the generalisation.
>   		 */
> -		if ((mask & smrs[i].mask) == mask &&
> -		    !((id ^ smrs[i].id) & ~smrs[i].mask))
> +
> +		if (smr_is_subset(&smrs[i], id, mask))
>   			return i;
> +
>   		/*
>   		 * If the new entry has any other overlap with an existing one,
>   		 * though, then there always exists at least one stream ID
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h
> index 703fd5817ec1..2e4f65412c6b 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h
> @@ -501,6 +501,11 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
>   		writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
>   }
> 
> +static inline bool smr_is_subset(struct arm_smmu_smr *smrs, u16 id, u16 mask)
> +{
> +	return (mask & smrs->mask) == mask && !((id ^ smrs->id) & ~smrs->mask);
> +}
> +
>   #define ARM_SMMU_GR0		0
>   #define ARM_SMMU_GR1		1
>   #define ARM_SMMU_CB(s, n)	((s)->numpage + (n))
> --
> 2.17.1
>

Dmitry Baryshkov Feb. 9, 2024, 10:53 a.m. UTC | #2

On Tue, 23 Jan 2024 at 16:46, Bibek Kumar Patro
<[e-mail]> wrote:
>
> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> the TLB to fetch just the next page table. MMU-500 features ACTLR
> register which is implementation defined and is used for Qualcomm SoCs
> to have a custom prefetch setting enabling TLB to prefetch the next set
> of page tables accordingly allowing for faster translations.
>
> ACTLR value is unique for each SMR (Stream matching register) and stored
> in a pre-populated table. This value is set to the register during
> context bank initialisation.
>
> Signed-off-by: Bibek Kumar Patro <[e-mail]>
> ---
>  drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 61 ++++++++++++++++++++++
>  drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 16 +++++-
>  drivers/iommu/arm/arm-smmu/arm-smmu.c      |  5 +-
>  drivers/iommu/arm/arm-smmu/arm-smmu.h      |  5 ++
>  4 files changed, 84 insertions(+), 3 deletions(-)


Reviewed-by: Dmitry Baryshkov <[e-mail]>

Rob Clark April 30, 2024, 7 p.m. UTC | #3

On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
<[e-mail]> wrote:
>
> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> the TLB to fetch just the next page table. MMU-500 features ACTLR
> register which is implementation defined and is used for Qualcomm SoCs
> to have a custom prefetch setting enabling TLB to prefetch the next set
> of page tables accordingly allowing for faster translations.
>
> ACTLR value is unique for each SMR (Stream matching register) and stored
> in a pre-populated table. This value is set to the register during
> context bank initialisation.
>
> Signed-off-by: Bibek Kumar Patro <[e-mail]>
> ---
>  drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 61 ++++++++++++++++++++++
>  drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 16 +++++-
>  drivers/iommu/arm/arm-smmu/arm-smmu.c      |  5 +-
>  drivers/iommu/arm/arm-smmu/arm-smmu.h      |  5 ++
>  4 files changed, 84 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> index 333daeb18c1c..6004c6d9a7b2 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> @@ -215,10 +215,42 @@ static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
>         return true;
>  }
>
> +static void qcom_smmu_set_actlr(struct device *dev, struct arm_smmu_device *smmu, int cbndx,
> +               const struct actlr_config *actlrcfg, const size_t num_actlrcfg)
> +{
> +       struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
> +       struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
> +       struct arm_smmu_smr *smr;
> +       u16 mask;
> +       int idx;
> +       u16 id;
> +       int i;
> +       int j;
> +
> +       for (i = 0; i < num_actlrcfg; i++) {
> +               id = actlrcfg[i].sid;
> +               mask = actlrcfg[i].mask;
> +
> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> +                       smr = &smmu->smrs[idx];
> +                       if (smr_is_subset(smr, id, mask)) {
> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> +                                               actlrcfg[i].actlr);

So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
downstream kgsl's PRR thing (which we'll need to implement vulkan
sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
to enable PRR.

        val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
        val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
        KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);

Any idea how this works?  And does it need to be done before or after
the ACTLR programming done in this patch?

BR,
-R

> +                               break;
> +                       }
> +               }
> +       }
> +}
> +
>  static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>                 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>  {
> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
> +       const struct actlr_variant *actlrvar;
> +       int cbndx = smmu_domain->cfg.cbndx;
>         struct adreno_smmu_priv *priv;
> +       int i;
>
>         smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
>
> @@ -248,6 +280,18 @@ static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>         priv->set_stall = qcom_adreno_smmu_set_stall;
>         priv->resume_translation = qcom_adreno_smmu_resume_translation;
>
> +       actlrvar = qsmmu->data->actlrvar;
> +       if (!actlrvar)
> +               return 0;
> +
> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
> +               if (actlrvar[i].io_start == smmu->ioaddr) {
> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
> +                                      actlrvar[i].num_actlrcfg);
> +                       break;
> +               }
> +       }
> +
>         return 0;
>  }
>
> @@ -274,7 +318,24 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
>  static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>                 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>  {
> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
> +       const struct actlr_variant *actlrvar;
> +       int cbndx = smmu_domain->cfg.cbndx;
> +       int i;
> +
>         smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
> +       actlrvar = qsmmu->data->actlrvar;
> +       if (!actlrvar)
> +               return 0;
> +
> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
> +               if (actlrvar[i].io_start == smmu->ioaddr) {
> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
> +                                      actlrvar[i].num_actlrcfg);
> +                       break;
> +               }
> +       }
>
>         return 0;
>  }
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> index f3b91963e234..3f651242de7c 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: GPL-2.0-only */
>  /*
> - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
> + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
>   */
>
>  #ifndef _ARM_SMMU_QCOM_H
> @@ -24,8 +24,22 @@ struct qcom_smmu_config {
>         const u32 *reg_offset;
>  };
>
> +struct actlr_config {
> +       u16 sid;
> +       u16 mask;
> +       u32 actlr;
> +};
> +
> +struct actlr_variant {
> +       const resource_size_t io_start;
> +       const struct actlr_config * const actlrcfg;
> +       const size_t num_actlrcfg;
> +};
> +
>  struct qcom_smmu_match_data {
> +       const struct actlr_variant * const actlrvar;
>         const struct qcom_smmu_config *cfg;
> +       const size_t num_smmu;
>         const struct arm_smmu_impl *impl;
>         const struct arm_smmu_impl *adreno_impl;
>  };
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c
> index d6d1a2a55cc0..0c7f700b27dd 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
> @@ -990,9 +990,10 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
>                  * expect simply identical entries for this case, but there's
>                  * no harm in accommodating the generalisation.
>                  */
> -               if ((mask & smrs[i].mask) == mask &&
> -                   !((id ^ smrs[i].id) & ~smrs[i].mask))
> +
> +               if (smr_is_subset(&smrs[i], id, mask))
>                         return i;
> +
>                 /*
>                  * If the new entry has any other overlap with an existing one,
>                  * though, then there always exists at least one stream ID
> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h
> index 703fd5817ec1..2e4f65412c6b 100644
> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h
> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h
> @@ -501,6 +501,11 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
>                 writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
>  }
>
> +static inline bool smr_is_subset(struct arm_smmu_smr *smrs, u16 id, u16 mask)
> +{
> +       return (mask & smrs->mask) == mask && !((id ^ smrs->id) & ~smrs->mask);
> +}
> +
>  #define ARM_SMMU_GR0           0
>  #define ARM_SMMU_GR1           1
>  #define ARM_SMMU_CB(s, n)      ((s)->numpage + (n))
> --
> 2.17.1
>
>

Bibek Kumar Patro May 10, 2024, 12:52 p.m. UTC | #4

On 5/1/2024 12:30 AM, Rob Clark wrote:
> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> <[e-mail]> wrote:
>>
>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>> register which is implementation defined and is used for Qualcomm SoCs
>> to have a custom prefetch setting enabling TLB to prefetch the next set
>> of page tables accordingly allowing for faster translations.
>>
>> ACTLR value is unique for each SMR (Stream matching register) and stored
>> in a pre-populated table. This value is set to the register during
>> context bank initialisation.
>>
>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>> ---
>>   drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 61 ++++++++++++++++++++++
>>   drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 16 +++++-
>>   drivers/iommu/arm/arm-smmu/arm-smmu.c      |  5 +-
>>   drivers/iommu/arm/arm-smmu/arm-smmu.h      |  5 ++
>>   4 files changed, 84 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
>> index 333daeb18c1c..6004c6d9a7b2 100644
>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
>> @@ -215,10 +215,42 @@ static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
>>          return true;
>>   }
>>
>> +static void qcom_smmu_set_actlr(struct device *dev, struct arm_smmu_device *smmu, int cbndx,
>> +               const struct actlr_config *actlrcfg, const size_t num_actlrcfg)
>> +{
>> +       struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
>> +       struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
>> +       struct arm_smmu_smr *smr;
>> +       u16 mask;
>> +       int idx;
>> +       u16 id;
>> +       int i;
>> +       int j;
>> +
>> +       for (i = 0; i < num_actlrcfg; i++) {
>> +               id = actlrcfg[i].sid;
>> +               mask = actlrcfg[i].mask;
>> +
>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>> +                       smr = &smmu->smrs[idx];
>> +                       if (smr_is_subset(smr, id, mask)) {
>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>> +                                               actlrcfg[i].actlr);
> 
> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> downstream kgsl's PRR thing (which we'll need to implement vulkan
> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> to enable PRR.
> 
>          val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>          val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>          KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> 
> Any idea how this works?  And does it need to be done before or after
> the ACTLR programming done in this patch?
> 
> BR,
> -R
> 

Hi Rob,

Can you please help provide some more clarification on the FIFO part? By 
FIFO are you referring to the storing of ACTLR data in the table?

Thanks for pointing to the downstream implementation of kgsl driver for
the PRR bit. Since kgsl driver is already handling this PRR bit's
setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
Thanks for bringing up this point.
I will send v10 patch series removing this BIT(5) setting from the ACTLR
table.

Thanks & regards,
Bibek

>> +                               break;
>> +                       }
>> +               }
>> +       }
>> +}
>> +
>>   static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>>                  struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>>   {
>> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
>> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
>> +       const struct actlr_variant *actlrvar;
>> +       int cbndx = smmu_domain->cfg.cbndx;
>>          struct adreno_smmu_priv *priv;
>> +       int i;
>>
>>          smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
>>
>> @@ -248,6 +280,18 @@ static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>>          priv->set_stall = qcom_adreno_smmu_set_stall;
>>          priv->resume_translation = qcom_adreno_smmu_resume_translation;
>>
>> +       actlrvar = qsmmu->data->actlrvar;
>> +       if (!actlrvar)
>> +               return 0;
>> +
>> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
>> +               if (actlrvar[i].io_start == smmu->ioaddr) {
>> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
>> +                                      actlrvar[i].num_actlrcfg);
>> +                       break;
>> +               }
>> +       }
>> +
>>          return 0;
>>   }
>>
>> @@ -274,7 +318,24 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
>>   static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>>                  struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>>   {
>> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
>> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
>> +       const struct actlr_variant *actlrvar;
>> +       int cbndx = smmu_domain->cfg.cbndx;
>> +       int i;
>> +
>>          smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
>> +       actlrvar = qsmmu->data->actlrvar;
>> +       if (!actlrvar)
>> +               return 0;
>> +
>> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
>> +               if (actlrvar[i].io_start == smmu->ioaddr) {
>> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
>> +                                      actlrvar[i].num_actlrcfg);
>> +                       break;
>> +               }
>> +       }
>>
>>          return 0;
>>   }
>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
>> index f3b91963e234..3f651242de7c 100644
>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
>> @@ -1,6 +1,6 @@
>>   /* SPDX-License-Identifier: GPL-2.0-only */
>>   /*
>> - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
>> + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
>>    */
>>
>>   #ifndef _ARM_SMMU_QCOM_H
>> @@ -24,8 +24,22 @@ struct qcom_smmu_config {
>>          const u32 *reg_offset;
>>   };
>>
>> +struct actlr_config {
>> +       u16 sid;
>> +       u16 mask;
>> +       u32 actlr;
>> +};
>> +
>> +struct actlr_variant {
>> +       const resource_size_t io_start;
>> +       const struct actlr_config * const actlrcfg;
>> +       const size_t num_actlrcfg;
>> +};
>> +
>>   struct qcom_smmu_match_data {
>> +       const struct actlr_variant * const actlrvar;
>>          const struct qcom_smmu_config *cfg;
>> +       const size_t num_smmu;
>>          const struct arm_smmu_impl *impl;
>>          const struct arm_smmu_impl *adreno_impl;
>>   };
>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c
>> index d6d1a2a55cc0..0c7f700b27dd 100644
>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
>> @@ -990,9 +990,10 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
>>                   * expect simply identical entries for this case, but there's
>>                   * no harm in accommodating the generalisation.
>>                   */
>> -               if ((mask & smrs[i].mask) == mask &&
>> -                   !((id ^ smrs[i].id) & ~smrs[i].mask))
>> +
>> +               if (smr_is_subset(&smrs[i], id, mask))
>>                          return i;
>> +
>>                  /*
>>                   * If the new entry has any other overlap with an existing one,
>>                   * though, then there always exists at least one stream ID
>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h
>> index 703fd5817ec1..2e4f65412c6b 100644
>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h
>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h
>> @@ -501,6 +501,11 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
>>                  writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
>>   }
>>
>> +static inline bool smr_is_subset(struct arm_smmu_smr *smrs, u16 id, u16 mask)
>> +{
>> +       return (mask & smrs->mask) == mask && !((id ^ smrs->id) & ~smrs->mask);
>> +}
>> +
>>   #define ARM_SMMU_GR0           0
>>   #define ARM_SMMU_GR1           1
>>   #define ARM_SMMU_CB(s, n)      ((s)->numpage + (n))
>> --
>> 2.17.1
>>
>>

Konrad Dybcio May 10, 2024, 1:02 p.m. UTC | #5

On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
> 
> 
> On 5/1/2024 12:30 AM, Rob Clark wrote:
>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>> <[e-mail]> wrote:
>>>
>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>> register which is implementation defined and is used for Qualcomm SoCs
>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>> of page tables accordingly allowing for faster translations.
>>>
>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>> in a pre-populated table. This value is set to the register during
>>> context bank initialisation.
>>>
>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>> ---

[...]

>>> +
>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>> +                       smr = &smmu->smrs[idx];
>>> +                       if (smr_is_subset(smr, id, mask)) {
>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>> +                                               actlrcfg[i].actlr);
>>
>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>> to enable PRR.
>>
>>          val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>          val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>          KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>
>> Any idea how this works?  And does it need to be done before or after
>> the ACTLR programming done in this patch?
>>
>> BR,
>> -R
>>
> 
> Hi Rob,
> 
> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
> 
> Thanks for pointing to the downstream implementation of kgsl driver for
> the PRR bit. Since kgsl driver is already handling this PRR bit's
> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.

The kgsl driver is not present upstream.

> Thanks for bringing up this point.
> I will send v10 patch series removing this BIT(5) setting from the ACTLR
> table.

I think it's generally saner to configure the SMMU from the SMMU driver..

Konrad

Rob Clark May 10, 2024, 7:48 p.m. UTC | #6

On Fri, May 10, 2024 at 5:52 AM Bibek Kumar Patro
<[e-mail]> wrote:
>
>
>
> On 5/1/2024 12:30 AM, Rob Clark wrote:
> > On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> > <[e-mail]> wrote:
> >>
> >> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> >> the TLB to fetch just the next page table. MMU-500 features ACTLR
> >> register which is implementation defined and is used for Qualcomm SoCs
> >> to have a custom prefetch setting enabling TLB to prefetch the next set
> >> of page tables accordingly allowing for faster translations.
> >>
> >> ACTLR value is unique for each SMR (Stream matching register) and stored
> >> in a pre-populated table. This value is set to the register during
> >> context bank initialisation.
> >>
> >> Signed-off-by: Bibek Kumar Patro <[e-mail]>
> >> ---
> >>   drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 61 ++++++++++++++++++++++
> >>   drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 16 +++++-
> >>   drivers/iommu/arm/arm-smmu/arm-smmu.c      |  5 +-
> >>   drivers/iommu/arm/arm-smmu/arm-smmu.h      |  5 ++
> >>   4 files changed, 84 insertions(+), 3 deletions(-)
> >>
> >> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> >> index 333daeb18c1c..6004c6d9a7b2 100644
> >> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> >> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
> >> @@ -215,10 +215,42 @@ static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
> >>          return true;
> >>   }
> >>
> >> +static void qcom_smmu_set_actlr(struct device *dev, struct arm_smmu_device *smmu, int cbndx,
> >> +               const struct actlr_config *actlrcfg, const size_t num_actlrcfg)
> >> +{
> >> +       struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
> >> +       struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
> >> +       struct arm_smmu_smr *smr;
> >> +       u16 mask;
> >> +       int idx;
> >> +       u16 id;
> >> +       int i;
> >> +       int j;
> >> +
> >> +       for (i = 0; i < num_actlrcfg; i++) {
> >> +               id = actlrcfg[i].sid;
> >> +               mask = actlrcfg[i].mask;
> >> +
> >> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> >> +                       smr = &smmu->smrs[idx];
> >> +                       if (smr_is_subset(smr, id, mask)) {
> >> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> >> +                                               actlrcfg[i].actlr);
> >
> > So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> > downstream kgsl's PRR thing (which we'll need to implement vulkan
> > sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> > to enable PRR.
> >
> >          val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
> >          val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
> >          KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> >
> > Any idea how this works?  And does it need to be done before or after
> > the ACTLR programming done in this patch?
> >
> > BR,
> > -R
> >
>
> Hi Rob,
>
> Can you please help provide some more clarification on the FIFO part? By
> FIFO are you referring to the storing of ACTLR data in the table?

Yeah, I mean it is writing the same ACTLR register multiple times to
program the table.  I'm wondering if that means we need to program the
table in a particular order compared to setting the PRR bit?  Like do
we need to program PRR bit first, or last?

I'm planning on adding an adreno_smmu_priv interface so that drm/msm
can call into arm-smmu-qcom to setup the PRR bit and the related
PRR_CFG_LADDR/PRR_CFG_UADDR registers.  And I'm just wondering if
there is an ordering constraint wrt. when qcom_smmu_set_actlr() is
called?

BR,
-R

> Thanks for pointing to the downstream implementation of kgsl driver for
> the PRR bit. Since kgsl driver is already handling this PRR bit's
> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> Thanks for bringing up this point.
> I will send v10 patch series removing this BIT(5) setting from the ACTLR
> table.
>
> Thanks & regards,
> Bibek
>
> >> +                               break;
> >> +                       }
> >> +               }
> >> +       }
> >> +}
> >> +
> >>   static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
> >>                  struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
> >>   {
> >> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
> >> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
> >> +       const struct actlr_variant *actlrvar;
> >> +       int cbndx = smmu_domain->cfg.cbndx;
> >>          struct adreno_smmu_priv *priv;
> >> +       int i;
> >>
> >>          smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
> >>
> >> @@ -248,6 +280,18 @@ static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
> >>          priv->set_stall = qcom_adreno_smmu_set_stall;
> >>          priv->resume_translation = qcom_adreno_smmu_resume_translation;
> >>
> >> +       actlrvar = qsmmu->data->actlrvar;
> >> +       if (!actlrvar)
> >> +               return 0;
> >> +
> >> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
> >> +               if (actlrvar[i].io_start == smmu->ioaddr) {
> >> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
> >> +                                      actlrvar[i].num_actlrcfg);
> >> +                       break;
> >> +               }
> >> +       }
> >> +
> >>          return 0;
> >>   }
> >>
> >> @@ -274,7 +318,24 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
> >>   static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
> >>                  struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
> >>   {
> >> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
> >> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
> >> +       const struct actlr_variant *actlrvar;
> >> +       int cbndx = smmu_domain->cfg.cbndx;
> >> +       int i;
> >> +
> >>          smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
> >> +       actlrvar = qsmmu->data->actlrvar;
> >> +       if (!actlrvar)
> >> +               return 0;
> >> +
> >> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
> >> +               if (actlrvar[i].io_start == smmu->ioaddr) {
> >> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
> >> +                                      actlrvar[i].num_actlrcfg);
> >> +                       break;
> >> +               }
> >> +       }
> >>
> >>          return 0;
> >>   }
> >> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> >> index f3b91963e234..3f651242de7c 100644
> >> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> >> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
> >> @@ -1,6 +1,6 @@
> >>   /* SPDX-License-Identifier: GPL-2.0-only */
> >>   /*
> >> - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
> >> + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
> >>    */
> >>
> >>   #ifndef _ARM_SMMU_QCOM_H
> >> @@ -24,8 +24,22 @@ struct qcom_smmu_config {
> >>          const u32 *reg_offset;
> >>   };
> >>
> >> +struct actlr_config {
> >> +       u16 sid;
> >> +       u16 mask;
> >> +       u32 actlr;
> >> +};
> >> +
> >> +struct actlr_variant {
> >> +       const resource_size_t io_start;
> >> +       const struct actlr_config * const actlrcfg;
> >> +       const size_t num_actlrcfg;
> >> +};
> >> +
> >>   struct qcom_smmu_match_data {
> >> +       const struct actlr_variant * const actlrvar;
> >>          const struct qcom_smmu_config *cfg;
> >> +       const size_t num_smmu;
> >>          const struct arm_smmu_impl *impl;
> >>          const struct arm_smmu_impl *adreno_impl;
> >>   };
> >> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c
> >> index d6d1a2a55cc0..0c7f700b27dd 100644
> >> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
> >> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
> >> @@ -990,9 +990,10 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
> >>                   * expect simply identical entries for this case, but there's
> >>                   * no harm in accommodating the generalisation.
> >>                   */
> >> -               if ((mask & smrs[i].mask) == mask &&
> >> -                   !((id ^ smrs[i].id) & ~smrs[i].mask))
> >> +
> >> +               if (smr_is_subset(&smrs[i], id, mask))
> >>                          return i;
> >> +
> >>                  /*
> >>                   * If the new entry has any other overlap with an existing one,
> >>                   * though, then there always exists at least one stream ID
> >> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h
> >> index 703fd5817ec1..2e4f65412c6b 100644
> >> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h
> >> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h
> >> @@ -501,6 +501,11 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
> >>                  writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
> >>   }
> >>
> >> +static inline bool smr_is_subset(struct arm_smmu_smr *smrs, u16 id, u16 mask)
> >> +{
> >> +       return (mask & smrs->mask) == mask && !((id ^ smrs->id) & ~smrs->mask);
> >> +}
> >> +
> >>   #define ARM_SMMU_GR0           0
> >>   #define ARM_SMMU_GR1           1
> >>   #define ARM_SMMU_CB(s, n)      ((s)->numpage + (n))
> >> --
> >> 2.17.1
> >>
> >>

Bibek Kumar Patro May 15, 2024, 1:59 p.m. UTC | #7

On 5/11/2024 1:18 AM, Rob Clark wrote:
> On Fri, May 10, 2024 at 5:52 AM Bibek Kumar Patro
> <[e-mail]> wrote:
>>
>>
>>
>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>> <[e-mail]> wrote:
>>>>
>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>> register which is implementation defined and is used for Qualcomm SoCs
>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>>> of page tables accordingly allowing for faster translations.
>>>>
>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>>> in a pre-populated table. This value is set to the register during
>>>> context bank initialisation.
>>>>
>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>> ---
>>>>    drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 61 ++++++++++++++++++++++
>>>>    drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h | 16 +++++-
>>>>    drivers/iommu/arm/arm-smmu/arm-smmu.c      |  5 +-
>>>>    drivers/iommu/arm/arm-smmu/arm-smmu.h      |  5 ++
>>>>    4 files changed, 84 insertions(+), 3 deletions(-)
>>>>
>>>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
>>>> index 333daeb18c1c..6004c6d9a7b2 100644
>>>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
>>>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
>>>> @@ -215,10 +215,42 @@ static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
>>>>           return true;
>>>>    }
>>>>
>>>> +static void qcom_smmu_set_actlr(struct device *dev, struct arm_smmu_device *smmu, int cbndx,
>>>> +               const struct actlr_config *actlrcfg, const size_t num_actlrcfg)
>>>> +{
>>>> +       struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
>>>> +       struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
>>>> +       struct arm_smmu_smr *smr;
>>>> +       u16 mask;
>>>> +       int idx;
>>>> +       u16 id;
>>>> +       int i;
>>>> +       int j;
>>>> +
>>>> +       for (i = 0; i < num_actlrcfg; i++) {
>>>> +               id = actlrcfg[i].sid;
>>>> +               mask = actlrcfg[i].mask;
>>>> +
>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>> +                       smr = &smmu->smrs[idx];
>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>>> +                                               actlrcfg[i].actlr);
>>>
>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>> to enable PRR.
>>>
>>>           val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>           val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>           KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>
>>> Any idea how this works?  And does it need to be done before or after
>>> the ACTLR programming done in this patch?
>>>
>>> BR,
>>> -R
>>>
>>
>> Hi Rob,
>>
>> Can you please help provide some more clarification on the FIFO part? By
>> FIFO are you referring to the storing of ACTLR data in the table?
> 
> Yeah, I mean it is writing the same ACTLR register multiple times to
> program the table.  I'm wondering if that means we need to program the
> table in a particular order compared to setting the PRR bit?  Like do
> we need to program PRR bit first, or last?
> 

Right Rob, this is the redundancy I am talking about, PRR bit's 
implementation is independent from SMMU ACTLR settings which should be
managed by KGSL driver.
We do not need to write the same register multiple times.

PRR bit can be independently programmed in any order wrt. to ACTLR table 
programming but for the case which is explained by you below I have 
suggested the order inline as per my understanding.

> I'm planning on adding an adreno_smmu_priv interface so that drm/msm
> can call into arm-smmu-qcom to setup the PRR bit and the related
> PRR_CFG_LADDR/PRR_CFG_UADDR registers.  And I'm just wondering if
> there is an ordering constraint wrt. when qcom_smmu_set_actlr() is
> called?

I see, thanks for clarification.
Since you are planning to add PRR bit + related registers'
configurations inside arm-smmu-qcom, then it would be better
to do it only after qcom_smmu_set_actlr() is called.

Reason being, if the PRR bit and related registers configuration is done
before hand, calling qcom_smmu_set_actlr() will just override other 
bits(including PRR bit) apart from the required bits since 
qcom_smmu_set_actlr() is doing only a write operation.

Thanks & regards,
Bibek

> 
> BR,
> -R
> 
>> Thanks for pointing to the downstream implementation of kgsl driver for
>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
>> Thanks for bringing up this point.
>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
>> table.
>>
>> Thanks & regards,
>> Bibek
>>
>>>> +                               break;
>>>> +                       }
>>>> +               }
>>>> +       }
>>>> +}
>>>> +
>>>>    static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>>>>                   struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>>>>    {
>>>> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
>>>> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
>>>> +       const struct actlr_variant *actlrvar;
>>>> +       int cbndx = smmu_domain->cfg.cbndx;
>>>>           struct adreno_smmu_priv *priv;
>>>> +       int i;
>>>>
>>>>           smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
>>>>
>>>> @@ -248,6 +280,18 @@ static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>>>>           priv->set_stall = qcom_adreno_smmu_set_stall;
>>>>           priv->resume_translation = qcom_adreno_smmu_resume_translation;
>>>>
>>>> +       actlrvar = qsmmu->data->actlrvar;
>>>> +       if (!actlrvar)
>>>> +               return 0;
>>>> +
>>>> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
>>>> +               if (actlrvar[i].io_start == smmu->ioaddr) {
>>>> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
>>>> +                                      actlrvar[i].num_actlrcfg);
>>>> +                       break;
>>>> +               }
>>>> +       }
>>>> +
>>>>           return 0;
>>>>    }
>>>>
>>>> @@ -274,7 +318,24 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
>>>>    static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
>>>>                   struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
>>>>    {
>>>> +       struct arm_smmu_device *smmu = smmu_domain->smmu;
>>>> +       struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
>>>> +       const struct actlr_variant *actlrvar;
>>>> +       int cbndx = smmu_domain->cfg.cbndx;
>>>> +       int i;
>>>> +
>>>>           smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
>>>> +       actlrvar = qsmmu->data->actlrvar;
>>>> +       if (!actlrvar)
>>>> +               return 0;
>>>> +
>>>> +       for (i = 0; i < qsmmu->data->num_smmu ; i++) {
>>>> +               if (actlrvar[i].io_start == smmu->ioaddr) {
>>>> +                       qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
>>>> +                                      actlrvar[i].num_actlrcfg);
>>>> +                       break;
>>>> +               }
>>>> +       }
>>>>
>>>>           return 0;
>>>>    }
>>>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
>>>> index f3b91963e234..3f651242de7c 100644
>>>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
>>>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
>>>> @@ -1,6 +1,6 @@
>>>>    /* SPDX-License-Identifier: GPL-2.0-only */
>>>>    /*
>>>> - * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
>>>> + * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
>>>>     */
>>>>
>>>>    #ifndef _ARM_SMMU_QCOM_H
>>>> @@ -24,8 +24,22 @@ struct qcom_smmu_config {
>>>>           const u32 *reg_offset;
>>>>    };
>>>>
>>>> +struct actlr_config {
>>>> +       u16 sid;
>>>> +       u16 mask;
>>>> +       u32 actlr;
>>>> +};
>>>> +
>>>> +struct actlr_variant {
>>>> +       const resource_size_t io_start;
>>>> +       const struct actlr_config * const actlrcfg;
>>>> +       const size_t num_actlrcfg;
>>>> +};
>>>> +
>>>>    struct qcom_smmu_match_data {
>>>> +       const struct actlr_variant * const actlrvar;
>>>>           const struct qcom_smmu_config *cfg;
>>>> +       const size_t num_smmu;
>>>>           const struct arm_smmu_impl *impl;
>>>>           const struct arm_smmu_impl *adreno_impl;
>>>>    };
>>>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c
>>>> index d6d1a2a55cc0..0c7f700b27dd 100644
>>>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
>>>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
>>>> @@ -990,9 +990,10 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
>>>>                    * expect simply identical entries for this case, but there's
>>>>                    * no harm in accommodating the generalisation.
>>>>                    */
>>>> -               if ((mask & smrs[i].mask) == mask &&
>>>> -                   !((id ^ smrs[i].id) & ~smrs[i].mask))
>>>> +
>>>> +               if (smr_is_subset(&smrs[i], id, mask))
>>>>                           return i;
>>>> +
>>>>                   /*
>>>>                    * If the new entry has any other overlap with an existing one,
>>>>                    * though, then there always exists at least one stream ID
>>>> diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h
>>>> index 703fd5817ec1..2e4f65412c6b 100644
>>>> --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h
>>>> +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h
>>>> @@ -501,6 +501,11 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
>>>>                   writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
>>>>    }
>>>>
>>>> +static inline bool smr_is_subset(struct arm_smmu_smr *smrs, u16 id, u16 mask)
>>>> +{
>>>> +       return (mask & smrs->mask) == mask && !((id ^ smrs->id) & ~smrs->mask);
>>>> +}
>>>> +
>>>>    #define ARM_SMMU_GR0           0
>>>>    #define ARM_SMMU_GR1           1
>>>>    #define ARM_SMMU_CB(s, n)      ((s)->numpage + (n))
>>>> --
>>>> 2.17.1
>>>>
>>>>

Bibek Kumar Patro May 15, 2024, 1:59 p.m. UTC | #8

On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
>>
>>
>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>> <[e-mail]> wrote:
>>>>
>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>> register which is implementation defined and is used for Qualcomm SoCs
>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>>> of page tables accordingly allowing for faster translations.
>>>>
>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>>> in a pre-populated table. This value is set to the register during
>>>> context bank initialisation.
>>>>
>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>> ---
> 
> [...]
> 
>>>> +
>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>> +                       smr = &smmu->smrs[idx];
>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>>> +                                               actlrcfg[i].actlr);
>>>
>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>> to enable PRR.
>>>
>>>           val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>           val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>           KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>
>>> Any idea how this works?  And does it need to be done before or after
>>> the ACTLR programming done in this patch?
>>>
>>> BR,
>>> -R
>>>
>>
>> Hi Rob,
>>
>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
>>
>> Thanks for pointing to the downstream implementation of kgsl driver for
>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> 
> The kgsl driver is not present upstream.
> 

Right kgsl is not present upstream, it would be better to avoid 
configuring the PRR bit and can be handled by kgsl directly in downstream.

>> Thanks for bringing up this point.
>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
>> table.
> 
> I think it's generally saner to configure the SMMU from the SMMU driver..

Yes, agree on this. But since PRR bit is not directly related to SMMU
configuration so I think it would be better to remove this PRR bit
setting from SMMU driver based on my understanding.


Thanks & regards,
Bibek
> 
> Konrad

Konrad Dybcio May 28, 2024, 12:59 p.m. UTC | #9

On 5/15/24 15:59, Bibek Kumar Patro wrote:
> 
> 
> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
>>>
>>>
>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>>> <[e-mail]> wrote:
>>>>>
>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>>> register which is implementation defined and is used for Qualcomm SoCs
>>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>>>> of page tables accordingly allowing for faster translations.
>>>>>
>>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>>>> in a pre-populated table. This value is set to the register during
>>>>> context bank initialisation.
>>>>>
>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>>> ---
>>
>> [...]
>>
>>>>> +
>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>>> +                       smr = &smmu->smrs[idx];
>>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>>>> +                                               actlrcfg[i].actlr);
>>>>
>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>>> to enable PRR.
>>>>
>>>>           val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>>           val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>>           KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>>
>>>> Any idea how this works?  And does it need to be done before or after
>>>> the ACTLR programming done in this patch?
>>>>
>>>> BR,
>>>> -R
>>>>
>>>
>>> Hi Rob,
>>>
>>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
>>>
>>> Thanks for pointing to the downstream implementation of kgsl driver for
>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
>>
>> The kgsl driver is not present upstream.
>>
> 
> Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.

No! Upstream is not a dumping ground to reduce your technical debt.

There is no kgsl driver upstream, so this ought to be handled here, in
the iommu driver (as poking at hardware A from driver B is usually not good
practice).

> 
>>> Thanks for bringing up this point.
>>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
>>> table.
>>
>> I think it's generally saner to configure the SMMU from the SMMU driver..
> 
> Yes, agree on this. But since PRR bit is not directly related to SMMU
> configuration so I think it would be better to remove this PRR bit
> setting from SMMU driver based on my understanding.

Why is it not related? We still don't know what it does.

Konrad

Dmitry Baryshkov May 28, 2024, 1:06 p.m. UTC | #10

On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
> 
> 
> On 5/15/24 15:59, Bibek Kumar Patro wrote:
> > 
> > 
> > On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
> > > On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
> > > > 
> > > > 
> > > > On 5/1/2024 12:30 AM, Rob Clark wrote:
> > > > > On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> > > > > <[e-mail]> wrote:
> > > > > > 
> > > > > > Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> > > > > > the TLB to fetch just the next page table. MMU-500 features ACTLR
> > > > > > register which is implementation defined and is used for Qualcomm SoCs
> > > > > > to have a custom prefetch setting enabling TLB to prefetch the next set
> > > > > > of page tables accordingly allowing for faster translations.
> > > > > > 
> > > > > > ACTLR value is unique for each SMR (Stream matching register) and stored
> > > > > > in a pre-populated table. This value is set to the register during
> > > > > > context bank initialisation.
> > > > > > 
> > > > > > Signed-off-by: Bibek Kumar Patro <[e-mail]>
> > > > > > ---
> > > 
> > > [...]
> > > 
> > > > > > +
> > > > > > +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> > > > > > +                       smr = &smmu->smrs[idx];
> > > > > > +                       if (smr_is_subset(smr, id, mask)) {
> > > > > > +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> > > > > > +                                               actlrcfg[i].actlr);
> > > > > 
> > > > > So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> > > > > downstream kgsl's PRR thing (which we'll need to implement vulkan
> > > > > sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> > > > > to enable PRR.
> > > > > 
> > > > >           val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
> > > > >           val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
> > > > >           KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> > > > > 
> > > > > Any idea how this works?  And does it need to be done before or after
> > > > > the ACTLR programming done in this patch?
> > > > > 
> > > > > BR,
> > > > > -R
> > > > > 
> > > > 
> > > > Hi Rob,
> > > > 
> > > > Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
> > > > 
> > > > Thanks for pointing to the downstream implementation of kgsl driver for
> > > > the PRR bit. Since kgsl driver is already handling this PRR bit's
> > > > setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> > > 
> > > The kgsl driver is not present upstream.
> > > 
> > 
> > Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
> 
> No! Upstream is not a dumping ground to reduce your technical debt.
> 
> There is no kgsl driver upstream, so this ought to be handled here, in
> the iommu driver (as poking at hardware A from driver B is usually not good
> practice).

I'd second the request here. If another driver has to control the
behaviour of another driver, please add corresponding API for that.

> 
> > 
> > > > Thanks for bringing up this point.
> > > > I will send v10 patch series removing this BIT(5) setting from the ACTLR
> > > > table.
> > > 
> > > I think it's generally saner to configure the SMMU from the SMMU driver..
> > 
> > Yes, agree on this. But since PRR bit is not directly related to SMMU
> > configuration so I think it would be better to remove this PRR bit
> > setting from SMMU driver based on my understanding.
> 
> Why is it not related? We still don't know what it does.
> 
> Konrad

Rob Clark May 28, 2024, 4:08 p.m. UTC | #11

On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
<[e-mail]> wrote:
>
> On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
> >
> >
> > On 5/15/24 15:59, Bibek Kumar Patro wrote:
> > >
> > >
> > > On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
> > > > On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
> > > > >
> > > > >
> > > > > On 5/1/2024 12:30 AM, Rob Clark wrote:
> > > > > > On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> > > > > > <[e-mail]> wrote:
> > > > > > >
> > > > > > > Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> > > > > > > the TLB to fetch just the next page table. MMU-500 features ACTLR
> > > > > > > register which is implementation defined and is used for Qualcomm SoCs
> > > > > > > to have a custom prefetch setting enabling TLB to prefetch the next set
> > > > > > > of page tables accordingly allowing for faster translations.
> > > > > > >
> > > > > > > ACTLR value is unique for each SMR (Stream matching register) and stored
> > > > > > > in a pre-populated table. This value is set to the register during
> > > > > > > context bank initialisation.
> > > > > > >
> > > > > > > Signed-off-by: Bibek Kumar Patro <[e-mail]>
> > > > > > > ---
> > > >
> > > > [...]
> > > >
> > > > > > > +
> > > > > > > +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> > > > > > > +                       smr = &smmu->smrs[idx];
> > > > > > > +                       if (smr_is_subset(smr, id, mask)) {
> > > > > > > +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> > > > > > > +                                               actlrcfg[i].actlr);
> > > > > >
> > > > > > So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> > > > > > downstream kgsl's PRR thing (which we'll need to implement vulkan
> > > > > > sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> > > > > > to enable PRR.
> > > > > >
> > > > > >           val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
> > > > > >           val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
> > > > > >           KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> > > > > >
> > > > > > Any idea how this works?  And does it need to be done before or after
> > > > > > the ACTLR programming done in this patch?
> > > > > >
> > > > > > BR,
> > > > > > -R
> > > > > >
> > > > >
> > > > > Hi Rob,
> > > > >
> > > > > Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
> > > > >
> > > > > Thanks for pointing to the downstream implementation of kgsl driver for
> > > > > the PRR bit. Since kgsl driver is already handling this PRR bit's
> > > > > setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> > > >
> > > > The kgsl driver is not present upstream.
> > > >
> > >
> > > Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
> >
> > No! Upstream is not a dumping ground to reduce your technical debt.
> >
> > There is no kgsl driver upstream, so this ought to be handled here, in
> > the iommu driver (as poking at hardware A from driver B is usually not good
> > practice).
>
> I'd second the request here. If another driver has to control the
> behaviour of another driver, please add corresponding API for that.

We have adreno_smmu_priv for this purpose ;-)

BR,
-R

> >
> > >
> > > > > Thanks for bringing up this point.
> > > > > I will send v10 patch series removing this BIT(5) setting from the ACTLR
> > > > > table.
> > > >
> > > > I think it's generally saner to configure the SMMU from the SMMU driver..
> > >
> > > Yes, agree on this. But since PRR bit is not directly related to SMMU
> > > configuration so I think it would be better to remove this PRR bit
> > > setting from SMMU driver based on my understanding.
> >
> > Why is it not related? We still don't know what it does.
> >
> > Konrad
>
> --
> With best wishes
> Dmitry

Dmitry Baryshkov May 28, 2024, 4:09 p.m. UTC | #12

On Tue, 28 May 2024 at 19:08, Rob Clark <[e-mail]> wrote:
>
> On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
> <[e-mail]> wrote:
> >
> > On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
> > >
> > >
> > > On 5/15/24 15:59, Bibek Kumar Patro wrote:
> > > >
> > > >
> > > > On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
> > > > > On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
> > > > > >
> > > > > >
> > > > > > On 5/1/2024 12:30 AM, Rob Clark wrote:
> > > > > > > On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> > > > > > > <[e-mail]> wrote:
> > > > > > > >
> > > > > > > > Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> > > > > > > > the TLB to fetch just the next page table. MMU-500 features ACTLR
> > > > > > > > register which is implementation defined and is used for Qualcomm SoCs
> > > > > > > > to have a custom prefetch setting enabling TLB to prefetch the next set
> > > > > > > > of page tables accordingly allowing for faster translations.
> > > > > > > >
> > > > > > > > ACTLR value is unique for each SMR (Stream matching register) and stored
> > > > > > > > in a pre-populated table. This value is set to the register during
> > > > > > > > context bank initialisation.
> > > > > > > >
> > > > > > > > Signed-off-by: Bibek Kumar Patro <[e-mail]>
> > > > > > > > ---
> > > > >
> > > > > [...]
> > > > >
> > > > > > > > +
> > > > > > > > +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> > > > > > > > +                       smr = &smmu->smrs[idx];
> > > > > > > > +                       if (smr_is_subset(smr, id, mask)) {
> > > > > > > > +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> > > > > > > > +                                               actlrcfg[i].actlr);
> > > > > > >
> > > > > > > So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> > > > > > > downstream kgsl's PRR thing (which we'll need to implement vulkan
> > > > > > > sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> > > > > > > to enable PRR.
> > > > > > >
> > > > > > >           val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
> > > > > > >           val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
> > > > > > >           KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> > > > > > >
> > > > > > > Any idea how this works?  And does it need to be done before or after
> > > > > > > the ACTLR programming done in this patch?
> > > > > > >
> > > > > > > BR,
> > > > > > > -R
> > > > > > >
> > > > > >
> > > > > > Hi Rob,
> > > > > >
> > > > > > Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
> > > > > >
> > > > > > Thanks for pointing to the downstream implementation of kgsl driver for
> > > > > > the PRR bit. Since kgsl driver is already handling this PRR bit's
> > > > > > setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> > > > >
> > > > > The kgsl driver is not present upstream.
> > > > >
> > > >
> > > > Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
> > >
> > > No! Upstream is not a dumping ground to reduce your technical debt.
> > >
> > > There is no kgsl driver upstream, so this ought to be handled here, in
> > > the iommu driver (as poking at hardware A from driver B is usually not good
> > > practice).
> >
> > I'd second the request here. If another driver has to control the
> > behaviour of another driver, please add corresponding API for that.
>
> We have adreno_smmu_priv for this purpose ;-)

Exactly

>
> BR,
> -R
>
> > >
> > > >
> > > > > > Thanks for bringing up this point.
> > > > > > I will send v10 patch series removing this BIT(5) setting from the ACTLR
> > > > > > table.
> > > > >
> > > > > I think it's generally saner to configure the SMMU from the SMMU driver..
> > > >
> > > > Yes, agree on this. But since PRR bit is not directly related to SMMU
> > > > configuration so I think it would be better to remove this PRR bit
> > > > setting from SMMU driver based on my understanding.
> > >
> > > Why is it not related? We still don't know what it does.
> > >
> > > Konrad
> >
> > --
> > With best wishes
> > Dmitry

Bibek Kumar Patro May 30, 2024, 9:21 a.m. UTC | #13

On 5/28/2024 6:29 PM, Konrad Dybcio wrote:
> 
> 
> On 5/15/24 15:59, Bibek Kumar Patro wrote:
>>
>>
>> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
>>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
>>>>
>>>>
>>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>>>> <[e-mail]> wrote:
>>>>>>
>>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which 
>>>>>> allows
>>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>>>> register which is implementation defined and is used for Qualcomm 
>>>>>> SoCs
>>>>>> to have a custom prefetch setting enabling TLB to prefetch the 
>>>>>> next set
>>>>>> of page tables accordingly allowing for faster translations.
>>>>>>
>>>>>> ACTLR value is unique for each SMR (Stream matching register) and 
>>>>>> stored
>>>>>> in a pre-populated table. This value is set to the register during
>>>>>> context bank initialisation.
>>>>>>
>>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>>>> ---
>>>
>>> [...]
>>>
>>>>>> +
>>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>>>> +                       smr = &smmu->smrs[idx];
>>>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>>>> +                               arm_smmu_cb_write(smmu, cbndx, 
>>>>>> ARM_SMMU_CB_ACTLR,
>>>>>> +                                               actlrcfg[i].actlr);
>>>>>
>>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>>>> to enable PRR.
>>>>>
>>>>>           val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>>>           val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>>>           KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>>>
>>>>> Any idea how this works?  And does it need to be done before or after
>>>>> the ACTLR programming done in this patch?
>>>>>
>>>>> BR,
>>>>> -R
>>>>>
>>>>
>>>> Hi Rob,
>>>>
>>>> Can you please help provide some more clarification on the FIFO 
>>>> part? By FIFO are you referring to the storing of ACTLR data in the 
>>>> table?
>>>>
>>>> Thanks for pointing to the downstream implementation of kgsl driver for
>>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
>>>
>>> The kgsl driver is not present upstream.
>>>
>>
>> Right kgsl is not present upstream, it would be better to avoid 
>> configuring the PRR bit and can be handled by kgsl directly in 
>> downstream.
> 
> No! Upstream is not a dumping ground to reduce your technical debt.
> 
> There is no kgsl driver upstream, so this ought to be handled here, in
> the iommu driver (as poking at hardware A from driver B is usually not good
> practice).
> 

Okay, so I see this point now. Driver B need to use hardware A's driver
exposed interface only to interact with the hardware functionality
instead of directly poking it. Agree on this, it looks to be an
appropriate approach.

>>
>>>> Thanks for bringing up this point.
>>>> I will send v10 patch series removing this BIT(5) setting from the 
>>>> ACTLR
>>>> table.
>>>
>>> I think it's generally saner to configure the SMMU from the SMMU 
>>> driver..
>>
>> Yes, agree on this. But since PRR bit is not directly related to SMMU
>> configuration so I think it would be better to remove this PRR bit
>> setting from SMMU driver based on my understanding.
> 
> Why is it not related? We still don't know what it does.
> 

By not related, I meant to say this bit is used for GFX implementation 
instead of direct SMMU related configuration.

Thanks & regards,
Bibek

> Konrad

Bibek Kumar Patro May 30, 2024, 9:21 a.m. UTC | #14

On 5/28/2024 6:36 PM, Dmitry Baryshkov wrote:
> On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
>>
>>
>> On 5/15/24 15:59, Bibek Kumar Patro wrote:
>>>
>>>
>>> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
>>>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
>>>>>
>>>>>
>>>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>>>>> <[e-mail]> wrote:
>>>>>>>
>>>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>>>>> register which is implementation defined and is used for Qualcomm SoCs
>>>>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>>>>>> of page tables accordingly allowing for faster translations.
>>>>>>>
>>>>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>>>>>> in a pre-populated table. This value is set to the register during
>>>>>>> context bank initialisation.
>>>>>>>
>>>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>>>>> ---
>>>>
>>>> [...]
>>>>
>>>>>>> +
>>>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>>>>> +                       smr = &smmu->smrs[idx];
>>>>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>>>>>> +                                               actlrcfg[i].actlr);
>>>>>>
>>>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>>>>> to enable PRR.
>>>>>>
>>>>>>            val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>>>>            val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>>>>            KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>>>>
>>>>>> Any idea how this works?  And does it need to be done before or after
>>>>>> the ACTLR programming done in this patch?
>>>>>>
>>>>>> BR,
>>>>>> -R
>>>>>>
>>>>>
>>>>> Hi Rob,
>>>>>
>>>>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
>>>>>
>>>>> Thanks for pointing to the downstream implementation of kgsl driver for
>>>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>>>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
>>>>
>>>> The kgsl driver is not present upstream.
>>>>
>>>
>>> Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
>>
>> No! Upstream is not a dumping ground to reduce your technical debt.
>>
>> There is no kgsl driver upstream, so this ought to be handled here, in
>> the iommu driver (as poking at hardware A from driver B is usually not good
>> practice).
> 
> I'd second the request here. If another driver has to control the
> behaviour of another driver, please add corresponding API for that.
> 

Ack, I understood this point now. Will add an interface for gfx to carry 
out the PRR bit implementation through smmu driver.

Thanks & regards,
Bibek

>>
>>>
>>>>> Thanks for bringing up this point.
>>>>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
>>>>> table.
>>>>
>>>> I think it's generally saner to configure the SMMU from the SMMU driver..
>>>
>>> Yes, agree on this. But since PRR bit is not directly related to SMMU
>>> configuration so I think it would be better to remove this PRR bit
>>> setting from SMMU driver based on my understanding.
>>
>> Why is it not related? We still don't know what it does.
>>
>> Konrad
>

Bibek Kumar Patro May 30, 2024, 9:21 a.m. UTC | #15

On 5/28/2024 9:38 PM, Rob Clark wrote:
> On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
> <[e-mail]> wrote:
>>
>> On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
>>>
>>>
>>> On 5/15/24 15:59, Bibek Kumar Patro wrote:
>>>>
>>>>
>>>> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
>>>>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
>>>>>>
>>>>>>
>>>>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>>>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>>>>>> <[e-mail]> wrote:
>>>>>>>>
>>>>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>>>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>>>>>> register which is implementation defined and is used for Qualcomm SoCs
>>>>>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>>>>>>> of page tables accordingly allowing for faster translations.
>>>>>>>>
>>>>>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>>>>>>> in a pre-populated table. This value is set to the register during
>>>>>>>> context bank initialisation.
>>>>>>>>
>>>>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>>>>>> ---
>>>>>
>>>>> [...]
>>>>>
>>>>>>>> +
>>>>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>>>>>> +                       smr = &smmu->smrs[idx];
>>>>>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>>>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>>>>>>> +                                               actlrcfg[i].actlr);
>>>>>>>
>>>>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>>>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>>>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>>>>>> to enable PRR.
>>>>>>>
>>>>>>>            val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>>>>>            val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>>>>>            KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>>>>>
>>>>>>> Any idea how this works?  And does it need to be done before or after
>>>>>>> the ACTLR programming done in this patch?
>>>>>>>
>>>>>>> BR,
>>>>>>> -R
>>>>>>>
>>>>>>
>>>>>> Hi Rob,
>>>>>>
>>>>>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
>>>>>>
>>>>>> Thanks for pointing to the downstream implementation of kgsl driver for
>>>>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>>>>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
>>>>>
>>>>> The kgsl driver is not present upstream.
>>>>>
>>>>
>>>> Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
>>>
>>> No! Upstream is not a dumping ground to reduce your technical debt.
>>>
>>> There is no kgsl driver upstream, so this ought to be handled here, in
>>> the iommu driver (as poking at hardware A from driver B is usually not good
>>> practice).
>>
>> I'd second the request here. If another driver has to control the
>> behaviour of another driver, please add corresponding API for that.
> 
> We have adreno_smmu_priv for this purpose ;-)
> 

Thanks Rob for pointing to this private interface structure between smmu
and gpu. I think it's similar to what you're trying to implement here
https://lore.kernel.org/all/[e-mail]/#t
I can add an api "set_actlr_prr()" with smmu_domain cookie, page pointer 
as two parameters. This api then can be used by drm/msm driver to carry 
out the prr implementation by simply calling this.
Would this be okay Rob,Konrad,Dmitry?
Let me know if any other suggestions you have in mind as well regarding
parameters and placement.

Thanks & regards,
Bibek

> BR,
> -R
> 
>>>
>>>>
>>>>>> Thanks for bringing up this point.
>>>>>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
>>>>>> table.
>>>>>
>>>>> I think it's generally saner to configure the SMMU from the SMMU driver..
>>>>
>>>> Yes, agree on this. But since PRR bit is not directly related to SMMU
>>>> configuration so I think it would be better to remove this PRR bit
>>>> setting from SMMU driver based on my understanding.
>>>
>>> Why is it not related? We still don't know what it does.
>>>
>>> Konrad
>>
>> --
>> With best wishes
>> Dmitry

Dmitry Baryshkov May 30, 2024, 11:18 a.m. UTC | #16

On Thu, May 30, 2024 at 02:51:56PM +0530, Bibek Kumar Patro wrote:
> 
> 
> On 5/28/2024 9:38 PM, Rob Clark wrote:
> > On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
> > <[e-mail]> wrote:
> > > 
> > > On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
> > > > 
> > > > 
> > > > On 5/15/24 15:59, Bibek Kumar Patro wrote:
> > > > > 
> > > > > 
> > > > > On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
> > > > > > On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
> > > > > > > 
> > > > > > > 
> > > > > > > On 5/1/2024 12:30 AM, Rob Clark wrote:
> > > > > > > > On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> > > > > > > > <[e-mail]> wrote:
> > > > > > > > > 
> > > > > > > > > Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> > > > > > > > > the TLB to fetch just the next page table. MMU-500 features ACTLR
> > > > > > > > > register which is implementation defined and is used for Qualcomm SoCs
> > > > > > > > > to have a custom prefetch setting enabling TLB to prefetch the next set
> > > > > > > > > of page tables accordingly allowing for faster translations.
> > > > > > > > > 
> > > > > > > > > ACTLR value is unique for each SMR (Stream matching register) and stored
> > > > > > > > > in a pre-populated table. This value is set to the register during
> > > > > > > > > context bank initialisation.
> > > > > > > > > 
> > > > > > > > > Signed-off-by: Bibek Kumar Patro <[e-mail]>
> > > > > > > > > ---
> > > > > > 
> > > > > > [...]
> > > > > > 
> > > > > > > > > +
> > > > > > > > > +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> > > > > > > > > +                       smr = &smmu->smrs[idx];
> > > > > > > > > +                       if (smr_is_subset(smr, id, mask)) {
> > > > > > > > > +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> > > > > > > > > +                                               actlrcfg[i].actlr);
> > > > > > > > 
> > > > > > > > So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> > > > > > > > downstream kgsl's PRR thing (which we'll need to implement vulkan
> > > > > > > > sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> > > > > > > > to enable PRR.
> > > > > > > > 
> > > > > > > >            val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
> > > > > > > >            val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
> > > > > > > >            KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> > > > > > > > 
> > > > > > > > Any idea how this works?  And does it need to be done before or after
> > > > > > > > the ACTLR programming done in this patch?
> > > > > > > > 
> > > > > > > > BR,
> > > > > > > > -R
> > > > > > > > 
> > > > > > > 
> > > > > > > Hi Rob,
> > > > > > > 
> > > > > > > Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
> > > > > > > 
> > > > > > > Thanks for pointing to the downstream implementation of kgsl driver for
> > > > > > > the PRR bit. Since kgsl driver is already handling this PRR bit's
> > > > > > > setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> > > > > > 
> > > > > > The kgsl driver is not present upstream.
> > > > > > 
> > > > > 
> > > > > Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
> > > > 
> > > > No! Upstream is not a dumping ground to reduce your technical debt.
> > > > 
> > > > There is no kgsl driver upstream, so this ought to be handled here, in
> > > > the iommu driver (as poking at hardware A from driver B is usually not good
> > > > practice).
> > > 
> > > I'd second the request here. If another driver has to control the
> > > behaviour of another driver, please add corresponding API for that.
> > 
> > We have adreno_smmu_priv for this purpose ;-)
> > 
> 
> Thanks Rob for pointing to this private interface structure between smmu
> and gpu. I think it's similar to what you're trying to implement here
> https://lore.kernel.org/all/[e-mail]/#t
> I can add an api "set_actlr_prr()" with smmu_domain cookie, page pointer as
> two parameters. This api then can be used by drm/msm driver to carry out the
> prr implementation by simply calling this.
> Would this be okay Rob,Konrad,Dmitry?

SGTM

> Let me know if any other suggestions you have in mind as well regarding
> parameters and placement.
> 
> Thanks & regards,
> Bibek
> 
> > BR,
> > -R
> > 
> > > > 
> > > > > 
> > > > > > > Thanks for bringing up this point.
> > > > > > > I will send v10 patch series removing this BIT(5) setting from the ACTLR
> > > > > > > table.
> > > > > > 
> > > > > > I think it's generally saner to configure the SMMU from the SMMU driver..
> > > > > 
> > > > > Yes, agree on this. But since PRR bit is not directly related to SMMU
> > > > > configuration so I think it would be better to remove this PRR bit
> > > > > setting from SMMU driver based on my understanding.
> > > > 
> > > > Why is it not related? We still don't know what it does.
> > > > 
> > > > Konrad
> > > 
> > > --
> > > With best wishes
> > > Dmitry

Rob Clark June 4, 2024, 6:49 p.m. UTC | #17

On Thu, May 30, 2024 at 2:22 AM Bibek Kumar Patro
<[e-mail]> wrote:
>
>
>
> On 5/28/2024 9:38 PM, Rob Clark wrote:
> > On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
> > <[e-mail]> wrote:
> >>
> >> On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
> >>>
> >>>
> >>> On 5/15/24 15:59, Bibek Kumar Patro wrote:
> >>>>
> >>>>
> >>>> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
> >>>>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
> >>>>>>
> >>>>>>
> >>>>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
> >>>>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> >>>>>>> <[e-mail]> wrote:
> >>>>>>>>
> >>>>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> >>>>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
> >>>>>>>> register which is implementation defined and is used for Qualcomm SoCs
> >>>>>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
> >>>>>>>> of page tables accordingly allowing for faster translations.
> >>>>>>>>
> >>>>>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
> >>>>>>>> in a pre-populated table. This value is set to the register during
> >>>>>>>> context bank initialisation.
> >>>>>>>>
> >>>>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
> >>>>>>>> ---
> >>>>>
> >>>>> [...]
> >>>>>
> >>>>>>>> +
> >>>>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> >>>>>>>> +                       smr = &smmu->smrs[idx];
> >>>>>>>> +                       if (smr_is_subset(smr, id, mask)) {
> >>>>>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> >>>>>>>> +                                               actlrcfg[i].actlr);
> >>>>>>>
> >>>>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> >>>>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
> >>>>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> >>>>>>> to enable PRR.
> >>>>>>>
> >>>>>>>            val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
> >>>>>>>            val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
> >>>>>>>            KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> >>>>>>>
> >>>>>>> Any idea how this works?  And does it need to be done before or after
> >>>>>>> the ACTLR programming done in this patch?
> >>>>>>>
> >>>>>>> BR,
> >>>>>>> -R
> >>>>>>>
> >>>>>>
> >>>>>> Hi Rob,
> >>>>>>
> >>>>>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
> >>>>>>
> >>>>>> Thanks for pointing to the downstream implementation of kgsl driver for
> >>>>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
> >>>>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> >>>>>
> >>>>> The kgsl driver is not present upstream.
> >>>>>
> >>>>
> >>>> Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
> >>>
> >>> No! Upstream is not a dumping ground to reduce your technical debt.
> >>>
> >>> There is no kgsl driver upstream, so this ought to be handled here, in
> >>> the iommu driver (as poking at hardware A from driver B is usually not good
> >>> practice).
> >>
> >> I'd second the request here. If another driver has to control the
> >> behaviour of another driver, please add corresponding API for that.
> >
> > We have adreno_smmu_priv for this purpose ;-)
> >
>
> Thanks Rob for pointing to this private interface structure between smmu
> and gpu. I think it's similar to what you're trying to implement here
> https://lore.kernel.org/all/[e-mail]/#t
> I can add an api "set_actlr_prr()" with smmu_domain cookie, page pointer
> as two parameters. This api then can be used by drm/msm driver to carry
> out the prr implementation by simply calling this.
> Would this be okay Rob,Konrad,Dmitry?
> Let me know if any other suggestions you have in mind as well regarding
> parameters and placement.

Hey Bibek, quick question.. is ACTLR preserved across a suspend/resume
cycle?  Or does it need to be reprogrammed on resume?  And same
question for these two PRR related regs:

  /* Global SMMU register offsets */
  #define KGSL_IOMMU_PRR_CFG_LADDR        0x6008
  #define KGSL_IOMMU_PRR_CFG_UADDR        0x600c

(ie. high/low 32b of the PRR page)

I was starting to type up a patch to add PRR configuration, but
depending on whether it interacts with suspend/resume, it might be
better form arm-smmu-qcom.c to just always enable and configure PRR
(including allocating a page to have an address to program into
PRR_CFG_LADDR/UADDR), and instead add an interface to return the PRR
page?  I think there is no harm in unconditionally configuring PRR for
gpu smmu.

BR,
-R

> Thanks & regards,
> Bibek
>
> > BR,
> > -R
> >
> >>>
> >>>>
> >>>>>> Thanks for bringing up this point.
> >>>>>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
> >>>>>> table.
> >>>>>
> >>>>> I think it's generally saner to configure the SMMU from the SMMU driver..
> >>>>
> >>>> Yes, agree on this. But since PRR bit is not directly related to SMMU
> >>>> configuration so I think it would be better to remove this PRR bit
> >>>> setting from SMMU driver based on my understanding.
> >>>
> >>> Why is it not related? We still don't know what it does.
> >>>
> >>> Konrad
> >>
> >> --
> >> With best wishes
> >> Dmitry

Bibek Kumar Patro June 5, 2024, 10:52 a.m. UTC | #18

On 6/5/2024 12:19 AM, Rob Clark wrote:
> On Thu, May 30, 2024 at 2:22 AM Bibek Kumar Patro
> <[e-mail]> wrote:
>>
>>
>>
>> On 5/28/2024 9:38 PM, Rob Clark wrote:
>>> On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
>>> <[e-mail]> wrote:
>>>>
>>>> On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
>>>>>
>>>>>
>>>>> On 5/15/24 15:59, Bibek Kumar Patro wrote:
>>>>>>
>>>>>>
>>>>>> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
>>>>>>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
>>>>>>>>
>>>>>>>>
>>>>>>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>>>>>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>>>>>>>> <[e-mail]> wrote:
>>>>>>>>>>
>>>>>>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>>>>>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>>>>>>>> register which is implementation defined and is used for Qualcomm SoCs
>>>>>>>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>>>>>>>>> of page tables accordingly allowing for faster translations.
>>>>>>>>>>
>>>>>>>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>>>>>>>>> in a pre-populated table. This value is set to the register during
>>>>>>>>>> context bank initialisation.
>>>>>>>>>>
>>>>>>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>>>>>>>> ---
>>>>>>>
>>>>>>> [...]
>>>>>>>
>>>>>>>>>> +
>>>>>>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>>>>>>>> +                       smr = &smmu->smrs[idx];
>>>>>>>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>>>>>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>>>>>>>>> +                                               actlrcfg[i].actlr);
>>>>>>>>>
>>>>>>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>>>>>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>>>>>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>>>>>>>> to enable PRR.
>>>>>>>>>
>>>>>>>>>             val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>>>>>>>             val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>>>>>>>             KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>>>>>>>
>>>>>>>>> Any idea how this works?  And does it need to be done before or after
>>>>>>>>> the ACTLR programming done in this patch?
>>>>>>>>>
>>>>>>>>> BR,
>>>>>>>>> -R
>>>>>>>>>
>>>>>>>>
>>>>>>>> Hi Rob,
>>>>>>>>
>>>>>>>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
>>>>>>>>
>>>>>>>> Thanks for pointing to the downstream implementation of kgsl driver for
>>>>>>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>>>>>>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
>>>>>>>
>>>>>>> The kgsl driver is not present upstream.
>>>>>>>
>>>>>>
>>>>>> Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
>>>>>
>>>>> No! Upstream is not a dumping ground to reduce your technical debt.
>>>>>
>>>>> There is no kgsl driver upstream, so this ought to be handled here, in
>>>>> the iommu driver (as poking at hardware A from driver B is usually not good
>>>>> practice).
>>>>
>>>> I'd second the request here. If another driver has to control the
>>>> behaviour of another driver, please add corresponding API for that.
>>>
>>> We have adreno_smmu_priv for this purpose ;-)
>>>
>>
>> Thanks Rob for pointing to this private interface structure between smmu
>> and gpu. I think it's similar to what you're trying to implement here
>> https://lore.kernel.org/all/[e-mail]/#t
>> I can add an api "set_actlr_prr()" with smmu_domain cookie, page pointer
>> as two parameters. This api then can be used by drm/msm driver to carry
>> out the prr implementation by simply calling this.
>> Would this be okay Rob,Konrad,Dmitry?
>> Let me know if any other suggestions you have in mind as well regarding
>> parameters and placement.
> 
> Hey Bibek, quick question.. is ACTLR preserved across a suspend/resume
> cycle?  Or does it need to be reprogrammed on resume?  And same
> question for these two PRR related regs:
> 
>    /* Global SMMU register offsets */
>    #define KGSL_IOMMU_PRR_CFG_LADDR        0x6008
>    #define KGSL_IOMMU_PRR_CFG_UADDR        0x600c
> 
> (ie. high/low 32b of the PRR page)
> 

Hey Rob, In suspend/resume, the register space power rails are not in
disabled state, so it won't go back to reset values and should retain
it's value. Only in hibernation cycle the registers' value would get reset.

So the hi/low address bit register for PRR page would also retain it's 
value along with the ACTLR registers.

> I was starting to type up a patch to add PRR configuration, but
> depending on whether it interacts with suspend/resume, it might be
> better form arm-smmu-qcom.c to just always enable and configure PRR
> (including allocating a page to have an address to program into
> PRR_CFG_LADDR/UADDR), and instead add an interface to return the PRR
> page?  I think there is no harm in unconditionally configuring PRR for
> gpu smmu.

Sounds okay though since this would not interact with suspend/resume path.
But I think, suppose in-case this page would have some other references 
as well before configuring the address to the registers for PRR 
configuration, then  GPU would be dependent on arm-smmu-qcom for this page.
So Instead an endpoint api in arm-smmu-qcom.c can recieve the just the 
page-address, and bit set status from drm/msm driver and can set/reset 
the bit along with any page-address they want ?
It would mean the interface will be smmu's , but the choice of 
configuration data to the registers' will be still with gpu.

I wrote up a small patch with this implementation, would you like to 
review that?
Will send it in this v11 series as new patch.

Thanks & regards,
Bibek

> 
> BR,
> -R
> 
>> Thanks & regards,
>> Bibek
>>
>>> BR,
>>> -R
>>>
>>>>>
>>>>>>
>>>>>>>> Thanks for bringing up this point.
>>>>>>>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
>>>>>>>> table.
>>>>>>>
>>>>>>> I think it's generally saner to configure the SMMU from the SMMU driver..
>>>>>>
>>>>>> Yes, agree on this. But since PRR bit is not directly related to SMMU
>>>>>> configuration so I think it would be better to remove this PRR bit
>>>>>> setting from SMMU driver based on my understanding.
>>>>>
>>>>> Why is it not related? We still don't know what it does.
>>>>>
>>>>> Konrad
>>>>
>>>> --
>>>> With best wishes
>>>> Dmitry

Rob Clark June 5, 2024, 10:13 p.m. UTC | #19

On Wed, Jun 5, 2024 at 3:52 AM Bibek Kumar Patro
<[e-mail]> wrote:
>
> On 6/5/2024 12:19 AM, Rob Clark wrote:
> > On Thu, May 30, 2024 at 2:22 AM Bibek Kumar Patro
> > <[e-mail]> wrote:
> >>
> >>
> >>
> >> On 5/28/2024 9:38 PM, Rob Clark wrote:
> >>> On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
> >>> <[e-mail]> wrote:
> >>>>
> >>>> On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
> >>>>>
> >>>>>
> >>>>> On 5/15/24 15:59, Bibek Kumar Patro wrote:
> >>>>>>
> >>>>>>
> >>>>>> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
> >>>>>>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
> >>>>>>>>
> >>>>>>>>
> >>>>>>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
> >>>>>>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
> >>>>>>>>> <[e-mail]> wrote:
> >>>>>>>>>>
> >>>>>>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
> >>>>>>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
> >>>>>>>>>> register which is implementation defined and is used for Qualcomm SoCs
> >>>>>>>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
> >>>>>>>>>> of page tables accordingly allowing for faster translations.
> >>>>>>>>>>
> >>>>>>>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
> >>>>>>>>>> in a pre-populated table. This value is set to the register during
> >>>>>>>>>> context bank initialisation.
> >>>>>>>>>>
> >>>>>>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
> >>>>>>>>>> ---
> >>>>>>>
> >>>>>>> [...]
> >>>>>>>
> >>>>>>>>>> +
> >>>>>>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
> >>>>>>>>>> +                       smr = &smmu->smrs[idx];
> >>>>>>>>>> +                       if (smr_is_subset(smr, id, mask)) {
> >>>>>>>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
> >>>>>>>>>> +                                               actlrcfg[i].actlr);
> >>>>>>>>>
> >>>>>>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
> >>>>>>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
> >>>>>>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
> >>>>>>>>> to enable PRR.
> >>>>>>>>>
> >>>>>>>>>             val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
> >>>>>>>>>             val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
> >>>>>>>>>             KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
> >>>>>>>>>
> >>>>>>>>> Any idea how this works?  And does it need to be done before or after
> >>>>>>>>> the ACTLR programming done in this patch?
> >>>>>>>>>
> >>>>>>>>> BR,
> >>>>>>>>> -R
> >>>>>>>>>
> >>>>>>>>
> >>>>>>>> Hi Rob,
> >>>>>>>>
> >>>>>>>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
> >>>>>>>>
> >>>>>>>> Thanks for pointing to the downstream implementation of kgsl driver for
> >>>>>>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
> >>>>>>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
> >>>>>>>
> >>>>>>> The kgsl driver is not present upstream.
> >>>>>>>
> >>>>>>
> >>>>>> Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
> >>>>>
> >>>>> No! Upstream is not a dumping ground to reduce your technical debt.
> >>>>>
> >>>>> There is no kgsl driver upstream, so this ought to be handled here, in
> >>>>> the iommu driver (as poking at hardware A from driver B is usually not good
> >>>>> practice).
> >>>>
> >>>> I'd second the request here. If another driver has to control the
> >>>> behaviour of another driver, please add corresponding API for that.
> >>>
> >>> We have adreno_smmu_priv for this purpose ;-)
> >>>
> >>
> >> Thanks Rob for pointing to this private interface structure between smmu
> >> and gpu. I think it's similar to what you're trying to implement here
> >> https://lore.kernel.org/all/[e-mail]/#t
> >> I can add an api "set_actlr_prr()" with smmu_domain cookie, page pointer
> >> as two parameters. This api then can be used by drm/msm driver to carry
> >> out the prr implementation by simply calling this.
> >> Would this be okay Rob,Konrad,Dmitry?
> >> Let me know if any other suggestions you have in mind as well regarding
> >> parameters and placement.
> >
> > Hey Bibek, quick question.. is ACTLR preserved across a suspend/resume
> > cycle?  Or does it need to be reprogrammed on resume?  And same
> > question for these two PRR related regs:
> >
> >    /* Global SMMU register offsets */
> >    #define KGSL_IOMMU_PRR_CFG_LADDR        0x6008
> >    #define KGSL_IOMMU_PRR_CFG_UADDR        0x600c
> >
> > (ie. high/low 32b of the PRR page)
> >
>
> Hey Rob, In suspend/resume, the register space power rails are not in
> disabled state, so it won't go back to reset values and should retain
> it's value. Only in hibernation cycle the registers' value would get reset.
>
> So the hi/low address bit register for PRR page would also retain it's
> value along with the ACTLR registers.
>
> > I was starting to type up a patch to add PRR configuration, but
> > depending on whether it interacts with suspend/resume, it might be
> > better form arm-smmu-qcom.c to just always enable and configure PRR
> > (including allocating a page to have an address to program into
> > PRR_CFG_LADDR/UADDR), and instead add an interface to return the PRR
> > page?  I think there is no harm in unconditionally configuring PRR for
> > gpu smmu.
>
> Sounds okay though since this would not interact with suspend/resume path.
> But I think, suppose in-case this page would have some other references
> as well before configuring the address to the registers for PRR
> configuration, then  GPU would be dependent on arm-smmu-qcom for this page.
> So Instead an endpoint api in arm-smmu-qcom.c can recieve the just the
> page-address, and bit set status from drm/msm driver and can set/reset
> the bit along with any page-address they want ?
> It would mean the interface will be smmu's , but the choice of
> configuration data to the registers' will be still with gpu.
>
> I wrote up a small patch with this implementation, would you like to
> review that?
> Will send it in this v11 series as new patch.

I think if there is no suspend/resume interaction, we should go back
to the original idea of page allocation in drm/msm.

Basically, I think the pros and cons are:

allocate in arm-smmu
  pro: easy to sequence programming with suspend/resume
  con: there isn't a convenient place to free the page on driver unload

allocate in drm/msm:
  pro: easy place to free the page in teardown
  con: harder to sequence with s/r

But if ACTLR and PRR_CFG_LADDR/UADDR are retained, then the con isn't
actually an issue ;-)

Anyways, I can type that patch..  the rest of drm/msm and userspace
changes (vm_bind + sparse) to get to the point where I can use PRR are
a somewhat bigger task so it will take me a while to get the point
where I can test any smmu patches.

BR,
-R


> Thanks & regards,
> Bibek
>
> >
> > BR,
> > -R
> >
> >> Thanks & regards,
> >> Bibek
> >>
> >>> BR,
> >>> -R
> >>>
> >>>>>
> >>>>>>
> >>>>>>>> Thanks for bringing up this point.
> >>>>>>>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
> >>>>>>>> table.
> >>>>>>>
> >>>>>>> I think it's generally saner to configure the SMMU from the SMMU driver..
> >>>>>>
> >>>>>> Yes, agree on this. But since PRR bit is not directly related to SMMU
> >>>>>> configuration so I think it would be better to remove this PRR bit
> >>>>>> setting from SMMU driver based on my understanding.
> >>>>>
> >>>>> Why is it not related? We still don't know what it does.
> >>>>>
> >>>>> Konrad
> >>>>
> >>>> --
> >>>> With best wishes
> >>>> Dmitry

Bibek Kumar Patro June 10, 2024, 10:12 a.m. UTC | #20

On 6/6/2024 3:43 AM, Rob Clark wrote:
> On Wed, Jun 5, 2024 at 3:52 AM Bibek Kumar Patro
> <[e-mail]> wrote:
>>
>> On 6/5/2024 12:19 AM, Rob Clark wrote:
>>> On Thu, May 30, 2024 at 2:22 AM Bibek Kumar Patro
>>> <[e-mail]> wrote:
>>>>
>>>>
>>>>
>>>> On 5/28/2024 9:38 PM, Rob Clark wrote:
>>>>> On Tue, May 28, 2024 at 6:06 AM Dmitry Baryshkov
>>>>> <[e-mail]> wrote:
>>>>>>
>>>>>> On Tue, May 28, 2024 at 02:59:51PM +0200, Konrad Dybcio wrote:
>>>>>>>
>>>>>>>
>>>>>>> On 5/15/24 15:59, Bibek Kumar Patro wrote:
>>>>>>>>
>>>>>>>>
>>>>>>>> On 5/10/2024 6:32 PM, Konrad Dybcio wrote:
>>>>>>>>> On 10.05.2024 2:52 PM, Bibek Kumar Patro wrote:
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> On 5/1/2024 12:30 AM, Rob Clark wrote:
>>>>>>>>>>> On Tue, Jan 23, 2024 at 7:00 AM Bibek Kumar Patro
>>>>>>>>>>> <[e-mail]> wrote:
>>>>>>>>>>>>
>>>>>>>>>>>> Currently in Qualcomm  SoCs the default prefetch is set to 1 which allows
>>>>>>>>>>>> the TLB to fetch just the next page table. MMU-500 features ACTLR
>>>>>>>>>>>> register which is implementation defined and is used for Qualcomm SoCs
>>>>>>>>>>>> to have a custom prefetch setting enabling TLB to prefetch the next set
>>>>>>>>>>>> of page tables accordingly allowing for faster translations.
>>>>>>>>>>>>
>>>>>>>>>>>> ACTLR value is unique for each SMR (Stream matching register) and stored
>>>>>>>>>>>> in a pre-populated table. This value is set to the register during
>>>>>>>>>>>> context bank initialisation.
>>>>>>>>>>>>
>>>>>>>>>>>> Signed-off-by: Bibek Kumar Patro <[e-mail]>
>>>>>>>>>>>> ---
>>>>>>>>>
>>>>>>>>> [...]
>>>>>>>>>
>>>>>>>>>>>> +
>>>>>>>>>>>> +               for_each_cfg_sme(cfg, fwspec, j, idx) {
>>>>>>>>>>>> +                       smr = &smmu->smrs[idx];
>>>>>>>>>>>> +                       if (smr_is_subset(smr, id, mask)) {
>>>>>>>>>>>> +                               arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
>>>>>>>>>>>> +                                               actlrcfg[i].actlr);
>>>>>>>>>>>
>>>>>>>>>>> So, this makes ACTLR look like kind of a FIFO.  But I'm looking at
>>>>>>>>>>> downstream kgsl's PRR thing (which we'll need to implement vulkan
>>>>>>>>>>> sparse residency), and it appears to be wanting to set BIT(5) in ACTLR
>>>>>>>>>>> to enable PRR.
>>>>>>>>>>>
>>>>>>>>>>>              val = KGSL_IOMMU_GET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR);
>>>>>>>>>>>              val |= FIELD_PREP(KGSL_IOMMU_ACTLR_PRR_ENABLE, 1);
>>>>>>>>>>>              KGSL_IOMMU_SET_CTX_REG(ctx, KGSL_IOMMU_CTX_ACTLR, val);
>>>>>>>>>>>
>>>>>>>>>>> Any idea how this works?  And does it need to be done before or after
>>>>>>>>>>> the ACTLR programming done in this patch?
>>>>>>>>>>>
>>>>>>>>>>> BR,
>>>>>>>>>>> -R
>>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> Hi Rob,
>>>>>>>>>>
>>>>>>>>>> Can you please help provide some more clarification on the FIFO part? By FIFO are you referring to the storing of ACTLR data in the table?
>>>>>>>>>>
>>>>>>>>>> Thanks for pointing to the downstream implementation of kgsl driver for
>>>>>>>>>> the PRR bit. Since kgsl driver is already handling this PRR bit's
>>>>>>>>>> setting, this makes setting the PRR BIT(5) by SMMU driver redundant.
>>>>>>>>>
>>>>>>>>> The kgsl driver is not present upstream.
>>>>>>>>>
>>>>>>>>
>>>>>>>> Right kgsl is not present upstream, it would be better to avoid configuring the PRR bit and can be handled by kgsl directly in downstream.
>>>>>>>
>>>>>>> No! Upstream is not a dumping ground to reduce your technical debt.
>>>>>>>
>>>>>>> There is no kgsl driver upstream, so this ought to be handled here, in
>>>>>>> the iommu driver (as poking at hardware A from driver B is usually not good
>>>>>>> practice).
>>>>>>
>>>>>> I'd second the request here. If another driver has to control the
>>>>>> behaviour of another driver, please add corresponding API for that.
>>>>>
>>>>> We have adreno_smmu_priv for this purpose ;-)
>>>>>
>>>>
>>>> Thanks Rob for pointing to this private interface structure between smmu
>>>> and gpu. I think it's similar to what you're trying to implement here
>>>> https://lore.kernel.org/all/[e-mail]/#t
>>>> I can add an api "set_actlr_prr()" with smmu_domain cookie, page pointer
>>>> as two parameters. This api then can be used by drm/msm driver to carry
>>>> out the prr implementation by simply calling this.
>>>> Would this be okay Rob,Konrad,Dmitry?
>>>> Let me know if any other suggestions you have in mind as well regarding
>>>> parameters and placement.
>>>
>>> Hey Bibek, quick question.. is ACTLR preserved across a suspend/resume
>>> cycle?  Or does it need to be reprogrammed on resume?  And same
>>> question for these two PRR related regs:
>>>
>>>     /* Global SMMU register offsets */
>>>     #define KGSL_IOMMU_PRR_CFG_LADDR        0x6008
>>>     #define KGSL_IOMMU_PRR_CFG_UADDR        0x600c
>>>
>>> (ie. high/low 32b of the PRR page)
>>>
>>
>> Hey Rob, In suspend/resume, the register space power rails are not in
>> disabled state, so it won't go back to reset values and should retain
>> it's value. Only in hibernation cycle the registers' value would get reset.
>>
>> So the hi/low address bit register for PRR page would also retain it's
>> value along with the ACTLR registers.
>>
>>> I was starting to type up a patch to add PRR configuration, but
>>> depending on whether it interacts with suspend/resume, it might be
>>> better form arm-smmu-qcom.c to just always enable and configure PRR
>>> (including allocating a page to have an address to program into
>>> PRR_CFG_LADDR/UADDR), and instead add an interface to return the PRR
>>> page?  I think there is no harm in unconditionally configuring PRR for
>>> gpu smmu.
>>
>> Sounds okay though since this would not interact with suspend/resume path.
>> But I think, suppose in-case this page would have some other references
>> as well before configuring the address to the registers for PRR
>> configuration, then  GPU would be dependent on arm-smmu-qcom for this page.
>> So Instead an endpoint api in arm-smmu-qcom.c can recieve the just the
>> page-address, and bit set status from drm/msm driver and can set/reset
>> the bit along with any page-address they want ?
>> It would mean the interface will be smmu's , but the choice of
>> configuration data to the registers' will be still with gpu.
>>
>> I wrote up a small patch with this implementation, would you like to
>> review that?
>> Will send it in this v11 series as new patch.
> 
> I think if there is no suspend/resume interaction, we should go back
> to the original idea of page allocation in drm/msm.
> 
> Basically, I think the pros and cons are:
> 
> allocate in arm-smmu
>    pro: easy to sequence programming with suspend/resume
>    con: there isn't a convenient place to free the page on driver unload
> 
> allocate in drm/msm:
>    pro: easy place to free the page in teardown
>    con: harder to sequence with s/r
> 
> But if ACTLR and PRR_CFG_LADDR/UADDR are retained, then the con isn't
> actually an issue ;-)
> 

Sounds right, also in this case the ownership of the page stays with
drm/msm which might also make it easy to handle the page for them.

> Anyways, I can type that patch..  the rest of drm/msm and userspace
> changes (vm_bind + sparse) to get to the point where I can use PRR are
> a somewhat bigger task so it will take me a while to get the point
> where I can test any smmu patches.
> 

Sure Rob get it. Previously in v11 I sent a patch adding a 
adreno-smmu-priv api with similar "page allocation in drm" design as you 
explained
above. Is that approach looking okay?
If it's okay can I add you in
suggested-by tag in that patch 
<https://lore.kernel.org/all/[e-mail]/> 
?

Thanks & regards,
Bibek


> BR,
> -R
> 
> 
>> Thanks & regards,
>> Bibek
>>
>>>
>>> BR,
>>> -R
>>>
>>>> Thanks & regards,
>>>> Bibek
>>>>
>>>>> BR,
>>>>> -R
>>>>>
>>>>>>>
>>>>>>>>
>>>>>>>>>> Thanks for bringing up this point.
>>>>>>>>>> I will send v10 patch series removing this BIT(5) setting from the ACTLR
>>>>>>>>>> table.
>>>>>>>>>
>>>>>>>>> I think it's generally saner to configure the SMMU from the SMMU driver..
>>>>>>>>
>>>>>>>> Yes, agree on this. But since PRR bit is not directly related to SMMU
>>>>>>>> configuration so I think it would be better to remove this PRR bit
>>>>>>>> setting from SMMU driver based on my understanding.
>>>>>>>
>>>>>>> Why is it not related? We still don't know what it does.
>>>>>>>
>>>>>>> Konrad
>>>>>>
>>>>>> --
>>>>>> With best wishes
>>>>>> Dmitry

Patch

diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
index 333daeb18c1c..6004c6d9a7b2 100644
--- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
+++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
@@ -215,10 +215,42 @@  static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
 	return true;
 }

+static void qcom_smmu_set_actlr(struct device *dev, struct arm_smmu_device *smmu, int cbndx,
+		const struct actlr_config *actlrcfg, const size_t num_actlrcfg)
+{
+	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct arm_smmu_smr *smr;
+	u16 mask;
+	int idx;
+	u16 id;
+	int i;
+	int j;
+
+	for (i = 0; i < num_actlrcfg; i++) {
+		id = actlrcfg[i].sid;
+		mask = actlrcfg[i].mask;
+
+		for_each_cfg_sme(cfg, fwspec, j, idx) {
+			smr = &smmu->smrs[idx];
+			if (smr_is_subset(smr, id, mask)) {
+				arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR,
+						actlrcfg[i].actlr);
+				break;
+			}
+		}
+	}
+}
+
 static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
 		struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
 {
+	struct arm_smmu_device *smmu = smmu_domain->smmu;
+	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
+	const struct actlr_variant *actlrvar;
+	int cbndx = smmu_domain->cfg.cbndx;
 	struct adreno_smmu_priv *priv;
+	int i;

 	smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;

@@ -248,6 +280,18 @@  static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
 	priv->set_stall = qcom_adreno_smmu_set_stall;
 	priv->resume_translation = qcom_adreno_smmu_resume_translation;

+	actlrvar = qsmmu->data->actlrvar;
+	if (!actlrvar)
+		return 0;
+
+	for (i = 0; i < qsmmu->data->num_smmu ; i++) {
+		if (actlrvar[i].io_start == smmu->ioaddr) {
+			qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
+				       actlrvar[i].num_actlrcfg);
+			break;
+		}
+	}
+
 	return 0;
 }

@@ -274,7 +318,24 @@  static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
 static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
 		struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
 {
+	struct arm_smmu_device *smmu = smmu_domain->smmu;
+	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
+	const struct actlr_variant *actlrvar;
+	int cbndx = smmu_domain->cfg.cbndx;
+	int i;
+
 	smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
+	actlrvar = qsmmu->data->actlrvar;
+	if (!actlrvar)
+		return 0;
+
+	for (i = 0; i < qsmmu->data->num_smmu ; i++) {
+		if (actlrvar[i].io_start == smmu->ioaddr) {
+			qcom_smmu_set_actlr(dev, smmu, cbndx, actlrvar[i].actlrcfg,
+				       actlrvar[i].num_actlrcfg);
+			break;
+		}
+	}

 	return 0;
 }
diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
index f3b91963e234..3f651242de7c 100644
--- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
+++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
@@ -1,6 +1,6 @@ 
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  */

 #ifndef _ARM_SMMU_QCOM_H
@@ -24,8 +24,22 @@  struct qcom_smmu_config {
 	const u32 *reg_offset;
 };

+struct actlr_config {
+	u16 sid;
+	u16 mask;
+	u32 actlr;
+};
+
+struct actlr_variant {
+	const resource_size_t io_start;
+	const struct actlr_config * const actlrcfg;
+	const size_t num_actlrcfg;
+};
+
 struct qcom_smmu_match_data {
+	const struct actlr_variant * const actlrvar;
 	const struct qcom_smmu_config *cfg;
+	const size_t num_smmu;
 	const struct arm_smmu_impl *impl;
 	const struct arm_smmu_impl *adreno_impl;
 };
diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c
index d6d1a2a55cc0..0c7f700b27dd 100644
--- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
+++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
@@ -990,9 +990,10 @@  static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
 		 * expect simply identical entries for this case, but there's
 		 * no harm in accommodating the generalisation.
 		 */
-		if ((mask & smrs[i].mask) == mask &&
-		    !((id ^ smrs[i].id) & ~smrs[i].mask))
+
+		if (smr_is_subset(&smrs[i], id, mask))
 			return i;
+
 		/*
 		 * If the new entry has any other overlap with an existing one,
 		 * though, then there always exists at least one stream ID
diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h
index 703fd5817ec1..2e4f65412c6b 100644
--- a/drivers/iommu/arm/arm-smmu/arm-smmu.h
+++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h
@@ -501,6 +501,11 @@  static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
 		writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
 }

+static inline bool smr_is_subset(struct arm_smmu_smr *smrs, u16 id, u16 mask)
+{
+	return (mask & smrs->mask) == mask && !((id ^ smrs->id) & ~smrs->mask);
+}
+
 #define ARM_SMMU_GR0		0
 #define ARM_SMMU_GR1		1
 #define ARM_SMMU_CB(s, n)	((s)->numpage + (n))