| Message ID | 32219cb9b058d7329ad8234a8a287701af1a0e34.1669176158.git.quic_asutoshd@quicinc.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Add Multi Circular Queue Support | expand |
On Tue, Nov 22, 2022 at 08:10:27PM -0800, Asutosh Das wrote: > Add support for completing requests from Completion Queue. > Some host controllers support vendor specific registers > that provide a bitmap of all CQ's which have at least one > completed CQE. Add this support. > The MCQ specification doesn't provide the Task Tag or its > equivalent in the Completion Queue Entry. > So use an indirect method to find the Task Tag from the > Completion Queue Entry. > > Co-developed-by: Can Guo <quic_cang@quicinc.com> > Signed-off-by: Can Guo <quic_cang@quicinc.com> > Signed-off-by: Asutosh Das <quic_asutoshd@quicinc.com> > Reported-by: kernel test robot <lkp@intel.com> What is this reported by for? > Reviewed-by: Bart Van Assche <bvanassche@acm.org> > --- > drivers/ufs/core/ufs-mcq.c | 57 ++++++++++++++++++++++++++++++++++++++++++ > drivers/ufs/core/ufshcd-priv.h | 43 +++++++++++++++++++++++++++++++ > drivers/ufs/core/ufshcd.c | 37 +++++++++++++++++++++++++++ > drivers/ufs/host/ufs-qcom.c | 16 ++++++++++++ > drivers/ufs/host/ufs-qcom.h | 4 +++ > include/ufs/ufshcd.h | 7 ++++++ > include/ufs/ufshci.h | 3 +++ > 7 files changed, 167 insertions(+) > > diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c > index 10a0d0d7..365ad98 100644 > --- a/drivers/ufs/core/ufs-mcq.c > +++ b/drivers/ufs/core/ufs-mcq.c > @@ -28,6 +28,7 @@ > ((((c) >> 16) & MCQ_QCFGPTR_MASK) * MCQ_QCFGPTR_UNIT) > #define MCQ_QCFG_SIZE 0x40 > #define MCQ_ENTRY_SIZE_IN_DWORD 8 > +#define CQE_UCD_BA GENMASK_ULL(63, 7) > > static int rw_queue_count_set(const char *val, const struct kernel_param *kp) > { > @@ -333,6 +334,59 @@ static void __iomem *mcq_opr_base(struct ufs_hba *hba, > return opr->base + opr->stride * i; > } > > +u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i) > +{ > + return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); > +} > + > +void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i) > +{ > + writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); > +} > + It'd be good to add a comment about tag extraction as you did in description. > +static int ufshcd_mcq_get_tag(struct ufs_hba *hba, > + struct ufs_hw_queue *hwq, > + struct cq_entry *cqe) > +{ > + dma_addr_t dma_addr; > + > + /* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */ > + BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0)); > + > + /* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */ > + dma_addr = le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA; > + > + return (dma_addr - hba->ucdl_dma_addr) / > + sizeof(struct utp_transfer_cmd_desc); > +} > + [...] > diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c > index dd53e85..89d29a1 100644 > --- a/drivers/ufs/host/ufs-qcom.c > +++ b/drivers/ufs/host/ufs-qcom.c > @@ -1454,6 +1454,21 @@ static int ufs_qcom_get_hba_mac(struct ufs_hba *hba) > return MAX_SUPP_MAC; > } > > +static int ufs_qcom_get_outstanding_cqs(struct ufs_hba *hba, > + unsigned long *ocqs) > +{ > + u32 cqis_vs; > + struct ufshcd_res_info *mcq_vs_res = &hba->res[RES_MCQ_VS]; > + > + if (!mcq_vs_res->base) > + return -EINVAL; > + > + cqis_vs = readl(mcq_vs_res->base + UFS_MEM_CQIS_VS); > + *ocqs = cqis_vs; Why can't you assign readl to *ocqs? Thanks, Mani > + > + return 0; > +} > + > /* > * struct ufs_hba_qcom_vops - UFS QCOM specific variant operations > * > @@ -1479,6 +1494,7 @@ static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = { > .program_key = ufs_qcom_ice_program_key, > .get_hba_mac = ufs_qcom_get_hba_mac, > .op_runtime_config = ufs_qcom_op_runtime_config, > + .get_outstanding_cqs = ufs_qcom_get_outstanding_cqs, > }; > > /** > diff --git a/drivers/ufs/host/ufs-qcom.h b/drivers/ufs/host/ufs-qcom.h > index 44466a3..7769f03 100644 > --- a/drivers/ufs/host/ufs-qcom.h > +++ b/drivers/ufs/host/ufs-qcom.h > @@ -72,6 +72,10 @@ enum { > UFS_UFS_DBG_RD_EDTL_RAM = 0x1900, > }; > > +enum { > + UFS_MEM_CQIS_VS = 0x8, > +}; > + > #define UFS_CNTLR_2_x_x_VEN_REGS_OFFSET(x) (0x000 + x) > #define UFS_CNTLR_3_x_x_VEN_REGS_OFFSET(x) (0x400 + x) > > diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h > index 853c525..d5fde64 100644 > --- a/include/ufs/ufshcd.h > +++ b/include/ufs/ufshcd.h > @@ -300,6 +300,7 @@ struct ufs_pwr_mode_info { > * @event_notify: called to notify important events > * @get_hba_mac: called to get vendor specific mac value, mandatory for mcq mode > * @op_runtime_config: called to config Operation and runtime regs Pointers > + * @get_outstanding_cqs: called to get outstanding completion queues > */ > struct ufs_hba_variant_ops { > const char *name; > @@ -340,6 +341,8 @@ struct ufs_hba_variant_ops { > enum ufs_event_type evt, void *data); > int (*get_hba_mac)(struct ufs_hba *hba); > int (*op_runtime_config)(struct ufs_hba *hba); > + int (*get_outstanding_cqs)(struct ufs_hba *hba, > + unsigned long *ocqs); > }; > > /* clock gating state */ > @@ -1064,6 +1067,8 @@ struct ufs_hba { > * @id: hardware queue ID > * @sq_tp_slot: current slot to which SQ tail pointer is pointing > * @sq_lock: serialize submission queue access > + * @cq_tail_slot: current slot to which CQ tail pointer is pointing > + * @cq_head_slot: current slot to which CQ head pointer is pointing > */ > struct ufs_hw_queue { > void __iomem *mcq_sq_head; > @@ -1079,6 +1084,8 @@ struct ufs_hw_queue { > u32 id; > u32 sq_tail_slot; > spinlock_t sq_lock; > + u32 cq_tail_slot; > + u32 cq_head_slot; > }; > > static inline bool is_mcq_enabled(struct ufs_hba *hba) > diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h > index c85bdf2..1e557ba 100644 > --- a/include/ufs/ufshci.h > +++ b/include/ufs/ufshci.h > @@ -262,6 +262,9 @@ enum { > /* UTMRLRSR - UTP Task Management Request Run-Stop Register 80h */ > #define UTP_TASK_REQ_LIST_RUN_STOP_BIT 0x1 > > +/* CQISy - CQ y Interrupt Status Register */ > +#define UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS 0x1 > + > /* UICCMD - UIC Command */ > #define COMMAND_OPCODE_MASK 0xFF > #define GEN_SELECTOR_INDEX_MASK 0xFFFF > -- > 2.7.4 >
On 11/28/22 09:00, Manivannan Sadhasivam wrote: > On Tue, Nov 22, 2022 at 08:10:27PM -0800, Asutosh Das wrote: >> Add support for completing requests from Completion Queue. >> Some host controllers support vendor specific registers >> that provide a bitmap of all CQ's which have at least one >> completed CQE. Add this support. >> The MCQ specification doesn't provide the Task Tag or its >> equivalent in the Completion Queue Entry. >> So use an indirect method to find the Task Tag from the >> Completion Queue Entry. >> >> Co-developed-by: Can Guo <quic_cang@quicinc.com> >> Signed-off-by: Can Guo <quic_cang@quicinc.com> >> Signed-off-by: Asutosh Das <quic_asutoshd@quicinc.com> >> Reported-by: kernel test robot <lkp@intel.com> > > What is this reported by for? I think that tag should be left out. "Reported-by: kernel test robot" should only be used for patches that fix bugs reported by the kernel test robot. I assume that a fix has been folded in into this patch that was reported by the kernel test robot. If that is the case, the "Reported-by: kernel test robot" tag is inappropriate. Bart.
diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 10a0d0d7..365ad98 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -28,6 +28,7 @@ ((((c) >> 16) & MCQ_QCFGPTR_MASK) * MCQ_QCFGPTR_UNIT) #define MCQ_QCFG_SIZE 0x40 #define MCQ_ENTRY_SIZE_IN_DWORD 8 +#define CQE_UCD_BA GENMASK_ULL(63, 7) static int rw_queue_count_set(const char *val, const struct kernel_param *kp) { @@ -333,6 +334,59 @@ static void __iomem *mcq_opr_base(struct ufs_hba *hba, return opr->base + opr->stride * i; } +u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i) +{ + return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); +} + +void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i) +{ + writel(val, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); +} + +static int ufshcd_mcq_get_tag(struct ufs_hba *hba, + struct ufs_hw_queue *hwq, + struct cq_entry *cqe) +{ + dma_addr_t dma_addr; + + /* sizeof(struct utp_transfer_cmd_desc) must be a multiple of 128 */ + BUILD_BUG_ON(sizeof(struct utp_transfer_cmd_desc) & GENMASK(6, 0)); + + /* Bits 63:7 UCD base address, 6:5 are reserved, 4:0 is SQ ID */ + dma_addr = le64_to_cpu(cqe->command_desc_base_addr) & CQE_UCD_BA; + + return (dma_addr - hba->ucdl_dma_addr) / + sizeof(struct utp_transfer_cmd_desc); +} + +static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, + struct ufs_hw_queue *hwq) +{ + struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); + int tag = ufshcd_mcq_get_tag(hba, hwq, cqe); + + ufshcd_compl_one_cqe(hba, tag, cqe); +} + +unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba, + struct ufs_hw_queue *hwq) +{ + unsigned long completed_reqs = 0; + + ufshcd_mcq_update_cq_tail_slot(hwq); + while (!ufshcd_mcq_is_cq_empty(hwq)) { + ufshcd_mcq_process_cqe(hba, hwq); + ufshcd_mcq_inc_cq_head_slot(hwq); + completed_reqs++; + } + + if (completed_reqs) + ufshcd_mcq_update_cq_head(hwq); + + return completed_reqs; +} + void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba) { struct ufs_hw_queue *hwq; @@ -376,6 +430,9 @@ void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba) hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP; hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP; + /* Reinitializing is needed upon HC reset */ + hwq->sq_tail_slot = hwq->cq_tail_slot = hwq->cq_head_slot = 0; + /* Enable Tail Entry Push Status interrupt only for non-poll queues */ if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]) writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE); diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index 6453449..c5b5bf3 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -69,6 +69,10 @@ int ufshcd_mcq_memory_alloc(struct ufs_hba *hba); void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba); void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds); void ufshcd_mcq_select_mcq_mode(struct ufs_hba *hba); +u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i); +void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i); +unsigned long ufshcd_mcq_poll_cqe_nolock(struct ufs_hba *hba, + struct ufs_hw_queue *hwq); struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, struct request *req); @@ -253,6 +257,15 @@ static inline int ufshcd_mcq_vops_op_runtime_config(struct ufs_hba *hba) return -EOPNOTSUPP; } +static inline int ufshcd_vops_get_outstanding_cqs(struct ufs_hba *hba, + unsigned long *ocqs) +{ + if (hba->vops && hba->vops->get_outstanding_cqs) + return hba->vops->get_outstanding_cqs(hba, ocqs); + + return -EOPNOTSUPP; +} + extern const struct ufs_pm_lvl_states ufs_pm_lvl_states[]; /** @@ -339,4 +352,34 @@ static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q) writel(val, q->mcq_sq_tail); } +static inline void ufshcd_mcq_update_cq_tail_slot(struct ufs_hw_queue *q) +{ + u32 val = readl(q->mcq_cq_tail); + + q->cq_tail_slot = val / sizeof(struct cq_entry); +} + +static inline bool ufshcd_mcq_is_cq_empty(struct ufs_hw_queue *q) +{ + return q->cq_head_slot == q->cq_tail_slot; +} + +static inline void ufshcd_mcq_inc_cq_head_slot(struct ufs_hw_queue *q) +{ + q->cq_head_slot++; + if (q->cq_head_slot == q->max_entries) + q->cq_head_slot = 0; +} + +static inline void ufshcd_mcq_update_cq_head(struct ufs_hw_queue *q) +{ + writel(q->cq_head_slot * sizeof(struct cq_entry), q->mcq_cq_head); +} + +static inline struct cq_entry *ufshcd_mcq_cur_cqe(struct ufs_hw_queue *q) +{ + struct cq_entry *cqe = q->cqe_base_addr; + + return cqe + q->cq_head_slot; +} #endif /* _UFSHCD_PRIV_H_ */ diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index f16d02c..7fb7c5f 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -6672,6 +6672,40 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) } /** + * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events + * @hba: per adapter instance + * + * Returns IRQ_HANDLED if interrupt is handled + */ +static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) +{ + struct ufs_hw_queue *hwq; + unsigned long outstanding_cqs; + unsigned int nr_queues; + int i, ret; + u32 events; + + ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); + if (ret) + outstanding_cqs = (1U << hba->nr_hw_queues) - 1; + + /* Exclude the poll queues */ + nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; + for_each_set_bit(i, &outstanding_cqs, nr_queues) { + hwq = &hba->uhq[i]; + + events = ufshcd_mcq_read_cqis(hba, i); + if (events) + ufshcd_mcq_write_cqis(hba, events, i); + + if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) + ufshcd_mcq_poll_cqe_nolock(hba, hwq); + } + + return IRQ_HANDLED; +} + +/** * ufshcd_sl_intr - Interrupt service routine * @hba: per adapter instance * @intr_status: contains interrupts generated by the controller @@ -6696,6 +6730,9 @@ static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) if (intr_status & UTP_TRANSFER_REQ_COMPL) retval |= ufshcd_transfer_req_compl(hba); + if (intr_status & MCQ_CQ_EVENT_STATUS) + retval |= ufshcd_handle_mcq_cq_events(hba); + return retval; } diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c index dd53e85..89d29a1 100644 --- a/drivers/ufs/host/ufs-qcom.c +++ b/drivers/ufs/host/ufs-qcom.c @@ -1454,6 +1454,21 @@ static int ufs_qcom_get_hba_mac(struct ufs_hba *hba) return MAX_SUPP_MAC; } +static int ufs_qcom_get_outstanding_cqs(struct ufs_hba *hba, + unsigned long *ocqs) +{ + u32 cqis_vs; + struct ufshcd_res_info *mcq_vs_res = &hba->res[RES_MCQ_VS]; + + if (!mcq_vs_res->base) + return -EINVAL; + + cqis_vs = readl(mcq_vs_res->base + UFS_MEM_CQIS_VS); + *ocqs = cqis_vs; + + return 0; +} + /* * struct ufs_hba_qcom_vops - UFS QCOM specific variant operations * @@ -1479,6 +1494,7 @@ static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = { .program_key = ufs_qcom_ice_program_key, .get_hba_mac = ufs_qcom_get_hba_mac, .op_runtime_config = ufs_qcom_op_runtime_config, + .get_outstanding_cqs = ufs_qcom_get_outstanding_cqs, }; /** diff --git a/drivers/ufs/host/ufs-qcom.h b/drivers/ufs/host/ufs-qcom.h index 44466a3..7769f03 100644 --- a/drivers/ufs/host/ufs-qcom.h +++ b/drivers/ufs/host/ufs-qcom.h @@ -72,6 +72,10 @@ enum { UFS_UFS_DBG_RD_EDTL_RAM = 0x1900, }; +enum { + UFS_MEM_CQIS_VS = 0x8, +}; + #define UFS_CNTLR_2_x_x_VEN_REGS_OFFSET(x) (0x000 + x) #define UFS_CNTLR_3_x_x_VEN_REGS_OFFSET(x) (0x400 + x) diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 853c525..d5fde64 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -300,6 +300,7 @@ struct ufs_pwr_mode_info { * @event_notify: called to notify important events * @get_hba_mac: called to get vendor specific mac value, mandatory for mcq mode * @op_runtime_config: called to config Operation and runtime regs Pointers + * @get_outstanding_cqs: called to get outstanding completion queues */ struct ufs_hba_variant_ops { const char *name; @@ -340,6 +341,8 @@ struct ufs_hba_variant_ops { enum ufs_event_type evt, void *data); int (*get_hba_mac)(struct ufs_hba *hba); int (*op_runtime_config)(struct ufs_hba *hba); + int (*get_outstanding_cqs)(struct ufs_hba *hba, + unsigned long *ocqs); }; /* clock gating state */ @@ -1064,6 +1067,8 @@ struct ufs_hba { * @id: hardware queue ID * @sq_tp_slot: current slot to which SQ tail pointer is pointing * @sq_lock: serialize submission queue access + * @cq_tail_slot: current slot to which CQ tail pointer is pointing + * @cq_head_slot: current slot to which CQ head pointer is pointing */ struct ufs_hw_queue { void __iomem *mcq_sq_head; @@ -1079,6 +1084,8 @@ struct ufs_hw_queue { u32 id; u32 sq_tail_slot; spinlock_t sq_lock; + u32 cq_tail_slot; + u32 cq_head_slot; }; static inline bool is_mcq_enabled(struct ufs_hba *hba) diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h index c85bdf2..1e557ba 100644 --- a/include/ufs/ufshci.h +++ b/include/ufs/ufshci.h @@ -262,6 +262,9 @@ enum { /* UTMRLRSR - UTP Task Management Request Run-Stop Register 80h */ #define UTP_TASK_REQ_LIST_RUN_STOP_BIT 0x1 +/* CQISy - CQ y Interrupt Status Register */ +#define UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS 0x1 + /* UICCMD - UIC Command */ #define COMMAND_OPCODE_MASK 0xFF #define GEN_SELECTOR_INDEX_MASK 0xFFFF