| Message ID | 20241108-ep-msi-v5-3-a14951c0d007@nxp.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | PCI: EP: Add RC-to-EP doorbell with platform MSI controller | expand |
On Fri, Nov 08, 2024 at 02:43:30PM -0500, Frank Li wrote: (snip) > +static void pci_epf_enable_doorbell(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) > +{ > + enum pci_barno bar = reg->doorbell_bar; > + struct pci_epf *epf = epf_test->epf; > + struct pci_epc *epc = epf->epc; > + struct pci_epf_bar db_bar; > + struct msi_msg *msg; > + u64 doorbell_addr; > + u32 align; > + int ret; > + > + align = epf_test->epc_features->align; > + align = align ? align : 128; > + > + if (epf_test->epc_features->bar[bar].type == BAR_FIXED) > + align = max(epf_test->epc_features->bar[bar].fixed_size, align); > + > + if (bar < BAR_0 || bar == epf_test->test_reg_bar || !epf->db_msg) { > + reg->status |= STATUS_DOORBELL_ENABLE_FAIL; > + return; > + } > + > + msg = &epf->db_msg[0].msg; > + doorbell_addr = msg->address_hi; > + doorbell_addr <<= 32; > + doorbell_addr |= msg->address_lo; > + > + db_bar.phys_addr = round_down(doorbell_addr, align); > + db_bar.barno = bar; > + db_bar.size = epf->bar[bar].size; > + db_bar.flags = epf->bar[bar].flags; > + db_bar.addr = NULL; Some of the code above ... > + > + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &db_bar); > + if (!ret) > + reg->status |= STATUS_DOORBELL_ENABLE_SUCCESS; > +} > + > @@ -909,12 +998,46 @@ static int pci_epf_test_bind(struct pci_epf *epf) > if (ret) > return ret; > > + ret = pci_epf_alloc_doorbell(epf, 1); > + if (!ret) { > + struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; > + struct msi_msg *msg = &epf->db_msg[0].msg; > + u32 align = epc_features->align; > + u64 doorbell_addr; > + enum pci_barno bar; > + > + bar = pci_epc_get_next_free_bar(epc_features, test_reg_bar + 1); > + > + ret = request_irq(epf->db_msg[0].virq, pci_epf_test_doorbell_handler, 0, > + "pci-test-doorbell", epf_test); > + if (ret) { > + dev_err(&epf->dev, > + "Failed to request irq %d, doorbell feature is not supported\n", > + epf->db_msg[0].virq); > + return 0; > + } > + > + align = align ? align : 128; > + > + if (epf_test->epc_features->bar[bar].type == BAR_FIXED) > + align = max(epf_test->epc_features->bar[bar].fixed_size, align); > + > + doorbell_addr = msg->address_hi; > + doorbell_addr <<= 32; > + doorbell_addr |= msg->address_lo; > + > + reg->doorbell_addr = doorbell_addr & (align - 1); > + reg->doorbell_data = msg->data; > + reg->doorbell_bar = bar; ... seems to be duplicated in this function. Perhaps create a helper function so that you don't need to duplicate it. Also one function is doing: reg->doorbell_addr = doorbell_addr & (align - 1); to align, the other one is doing: round_down(doorbell_addr, align); Which seems to be a bit inconsistent. Anyway, if you move this to a helper, they will both use the same way to align the address. (May I suggest that you use ALIGN_DOWN() in the helper.) Kind regards, Niklas
On Mon, Nov 11, 2024 at 03:43:12PM +0100, Niklas Cassel wrote: > On Fri, Nov 08, 2024 at 02:43:30PM -0500, Frank Li wrote: > > Perhaps create a helper function so that you don't need to duplicate it. Something like this on top of your series: diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 8ede7aded03ee..b1707b4425432 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -656,16 +656,25 @@ static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test, } } -static void pci_epf_enable_doorbell(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) +static int pci_epf_get_doorbell_addr(struct pci_epf_test *epf_test, + enum pci_barno bar, u64 *db_base, + u64 *db_offset) { - enum pci_barno bar = reg->doorbell_bar; struct pci_epf *epf = epf_test->epf; - struct pci_epc *epc = epf->epc; - struct pci_epf_bar db_bar; struct msi_msg *msg; - u64 doorbell_addr; + u64 doorbell_addr, mask; u32 align; - int ret; + + if (bar < BAR_0 || bar == epf_test->test_reg_bar || !epf->db_msg) + return -EINVAL; + + msg = &epf->db_msg[0].msg; + doorbell_addr = msg->address_hi; + doorbell_addr <<= 32; + doorbell_addr |= msg->address_lo; + + if (!doorbell_addr) + return -EINVAL; align = epf_test->epc_features->align; align = align ? align : 128; @@ -673,17 +682,28 @@ static void pci_epf_enable_doorbell(struct pci_epf_test *epf_test, struct pci_ep if (epf_test->epc_features->bar[bar].type == BAR_FIXED) align = max(epf_test->epc_features->bar[bar].fixed_size, align); - if (bar < BAR_0 || bar == epf_test->test_reg_bar || !epf->db_msg) { + mask = align - 1; + *db_base = doorbell_addr & ~mask; + *db_offset = doorbell_addr & mask; + + return 0; +} + +static void pci_epf_enable_doorbell(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) +{ + enum pci_barno bar = reg->doorbell_bar; + struct pci_epf *epf = epf_test->epf; + struct pci_epc *epc = epf->epc; + struct pci_epf_bar db_bar; + u64 db_base, db_offset; + int ret; + + if (pci_epf_get_doorbell_addr(epf_test, bar, &db_base, &db_offset)) { reg->status |= STATUS_DOORBELL_ENABLE_FAIL; return; } - msg = &epf->db_msg[0].msg; - doorbell_addr = msg->address_hi; - doorbell_addr <<= 32; - doorbell_addr |= msg->address_lo; - - db_bar.phys_addr = round_down(doorbell_addr, align); + db_bar.phys_addr = db_base; db_bar.barno = bar; db_bar.size = epf->bar[bar].size; db_bar.flags = epf->bar[bar].flags; @@ -1015,9 +1035,7 @@ static int pci_epf_test_bind(struct pci_epf *epf) ret = pci_epf_alloc_doorbell(epf, 1); if (!ret) { struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; - struct msi_msg *msg = &epf->db_msg[0].msg; - u32 align = epc_features->align; - u64 doorbell_addr; + u64 db_base, db_offset; enum pci_barno bar; bar = pci_epc_get_next_free_bar(epc_features, test_reg_bar + 1); @@ -1031,17 +1049,15 @@ static int pci_epf_test_bind(struct pci_epf *epf) return 0; } - align = align ? align : 128; - - if (epf_test->epc_features->bar[bar].type == BAR_FIXED) - align = max(epf_test->epc_features->bar[bar].fixed_size, align); - - doorbell_addr = msg->address_hi; - doorbell_addr <<= 32; - doorbell_addr |= msg->address_lo; + if (pci_epf_get_doorbell_addr(epf_test, bar, &db_base, + &db_offset)) { + dev_err(&epf->dev, "Failed to get doorbell address\n"); + free_irq(epf->db_msg[0].virq, epf_test); + return 0; + } - reg->doorbell_addr = doorbell_addr & (align - 1); - reg->doorbell_data = msg->data; + reg->doorbell_addr = db_offset; + reg->doorbell_data = epf->db_msg[0].msg.data; reg->doorbell_bar = bar; } > > Also one function is doing: > reg->doorbell_addr = doorbell_addr & (align - 1); > > to align, the other one is doing: > round_down(doorbell_addr, align); > > Which seems to be a bit inconsistent. I now see why you did this. One function is using the db offset, and the other is using the db base. I strongly suggest that you rename: reg->doorbell_addr to reg->doorbell_offset and PCI_ENDPOINT_TEST_DB_ADDR to PCI_ENDPOINT_TEST_DB_OFFSET since the current names are very confusing. Kind regards, Niklas
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 7c2ed6eae53ad..52e36c7c04c85 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -11,12 +11,14 @@ #include <linux/dmaengine.h> #include <linux/io.h> #include <linux/module.h> +#include <linux/msi.h> #include <linux/slab.h> #include <linux/pci_ids.h> #include <linux/random.h> #include <linux/pci-epc.h> #include <linux/pci-epf.h> +#include <linux/pci-ep-msi.h> #include <linux/pci_regs.h> #define IRQ_TYPE_INTX 0 @@ -29,6 +31,8 @@ #define COMMAND_READ BIT(3) #define COMMAND_WRITE BIT(4) #define COMMAND_COPY BIT(5) +#define COMMAND_ENABLE_DOORBELL BIT(6) +#define COMMAND_DISABLE_DOORBELL BIT(7) #define STATUS_READ_SUCCESS BIT(0) #define STATUS_READ_FAIL BIT(1) @@ -39,6 +43,11 @@ #define STATUS_IRQ_RAISED BIT(6) #define STATUS_SRC_ADDR_INVALID BIT(7) #define STATUS_DST_ADDR_INVALID BIT(8) +#define STATUS_DOORBELL_SUCCESS BIT(9) +#define STATUS_DOORBELL_ENABLE_SUCCESS BIT(10) +#define STATUS_DOORBELL_ENABLE_FAIL BIT(11) +#define STATUS_DOORBELL_DISABLE_SUCCESS BIT(12) +#define STATUS_DOORBELL_DISABLE_FAIL BIT(13) #define FLAG_USE_DMA BIT(0) @@ -74,6 +83,9 @@ struct pci_epf_test_reg { u32 irq_type; u32 irq_number; u32 flags; + u32 doorbell_bar; + u32 doorbell_addr; + u32 doorbell_data; } __packed; static struct pci_epf_header test_header = { @@ -630,6 +642,63 @@ static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test, } } +static void pci_epf_enable_doorbell(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) +{ + enum pci_barno bar = reg->doorbell_bar; + struct pci_epf *epf = epf_test->epf; + struct pci_epc *epc = epf->epc; + struct pci_epf_bar db_bar; + struct msi_msg *msg; + u64 doorbell_addr; + u32 align; + int ret; + + align = epf_test->epc_features->align; + align = align ? align : 128; + + if (epf_test->epc_features->bar[bar].type == BAR_FIXED) + align = max(epf_test->epc_features->bar[bar].fixed_size, align); + + if (bar < BAR_0 || bar == epf_test->test_reg_bar || !epf->db_msg) { + reg->status |= STATUS_DOORBELL_ENABLE_FAIL; + return; + } + + msg = &epf->db_msg[0].msg; + doorbell_addr = msg->address_hi; + doorbell_addr <<= 32; + doorbell_addr |= msg->address_lo; + + db_bar.phys_addr = round_down(doorbell_addr, align); + db_bar.barno = bar; + db_bar.size = epf->bar[bar].size; + db_bar.flags = epf->bar[bar].flags; + db_bar.addr = NULL; + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &db_bar); + if (!ret) + reg->status |= STATUS_DOORBELL_ENABLE_SUCCESS; +} + +static void pci_epf_disable_doorbell(struct pci_epf_test *epf_test, struct pci_epf_test_reg *reg) +{ + enum pci_barno bar = reg->doorbell_bar; + struct pci_epf *epf = epf_test->epf; + struct pci_epc *epc = epf->epc; + int ret; + + if (bar < BAR_0 || bar == epf_test->test_reg_bar || !epf->db_msg) { + reg->status |= STATUS_DOORBELL_DISABLE_FAIL; + return; + } + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[bar]); + if (ret) + reg->status |= STATUS_DOORBELL_DISABLE_FAIL; + else + reg->status |= STATUS_DOORBELL_DISABLE_SUCCESS; +} + static void pci_epf_test_cmd_handler(struct work_struct *work) { u32 command; @@ -676,6 +745,14 @@ static void pci_epf_test_cmd_handler(struct work_struct *work) pci_epf_test_copy(epf_test, reg); pci_epf_test_raise_irq(epf_test, reg); break; + case COMMAND_ENABLE_DOORBELL: + pci_epf_enable_doorbell(epf_test, reg); + pci_epf_test_raise_irq(epf_test, reg); + break; + case COMMAND_DISABLE_DOORBELL: + pci_epf_disable_doorbell(epf_test, reg); + pci_epf_test_raise_irq(epf_test, reg); + break; default: dev_err(dev, "Invalid command 0x%x\n", command); break; @@ -810,6 +887,18 @@ static int pci_epf_test_link_down(struct pci_epf *epf) return 0; } +static irqreturn_t pci_epf_test_doorbell_handler(int irq, void *data) +{ + struct pci_epf_test *epf_test = data; + enum pci_barno test_reg_bar = epf_test->test_reg_bar; + struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; + + reg->status |= STATUS_DOORBELL_SUCCESS; + pci_epf_test_raise_irq(epf_test, reg); + + return IRQ_HANDLED; +} + static const struct pci_epc_event_ops pci_epf_test_event_ops = { .epc_init = pci_epf_test_epc_init, .epc_deinit = pci_epf_test_epc_deinit, @@ -909,12 +998,46 @@ static int pci_epf_test_bind(struct pci_epf *epf) if (ret) return ret; + ret = pci_epf_alloc_doorbell(epf, 1); + if (!ret) { + struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; + struct msi_msg *msg = &epf->db_msg[0].msg; + u32 align = epc_features->align; + u64 doorbell_addr; + enum pci_barno bar; + + bar = pci_epc_get_next_free_bar(epc_features, test_reg_bar + 1); + + ret = request_irq(epf->db_msg[0].virq, pci_epf_test_doorbell_handler, 0, + "pci-test-doorbell", epf_test); + if (ret) { + dev_err(&epf->dev, + "Failed to request irq %d, doorbell feature is not supported\n", + epf->db_msg[0].virq); + return 0; + } + + align = align ? align : 128; + + if (epf_test->epc_features->bar[bar].type == BAR_FIXED) + align = max(epf_test->epc_features->bar[bar].fixed_size, align); + + doorbell_addr = msg->address_hi; + doorbell_addr <<= 32; + doorbell_addr |= msg->address_lo; + + reg->doorbell_addr = doorbell_addr & (align - 1); + reg->doorbell_data = msg->data; + reg->doorbell_bar = bar; + } + return 0; } static void pci_epf_test_unbind(struct pci_epf *epf) { struct pci_epf_test *epf_test = epf_get_drvdata(epf); + struct pci_epf_test_reg *reg = epf_test->reg[epf_test->test_reg_bar]; struct pci_epc *epc = epf->epc; cancel_delayed_work(&epf_test->cmd_handler); @@ -922,6 +1045,12 @@ static void pci_epf_test_unbind(struct pci_epf *epf) pci_epf_test_clean_dma_chan(epf_test); pci_epf_test_clear_bar(epf); } + + if (reg->doorbell_bar > 0) { + free_irq(epf->db_msg[0].virq, epf_test); + pci_epf_free_doorbell(epf); + } + pci_epf_test_free_space(epf); }
Add three registers: doorbell_bar, doorbell_addr, and doorbell_data, along with doorbell_done. Use pci_epf_alloc_doorbell() to allocate a doorbell address space. Enable the Root Complex (RC) side driver to trigger pci-epc-test's doorbell callback handler by writing doorbell_data to the mapped doorbell_bar's address space. Set doorbell_done in the doorbell callback to indicate completion. To avoid broken compatibility, add new command COMMAND_ENABLE_DOORBELL and COMMAND_DISABLE_DOORBELL. Host side need send COMMAND_ENABLE_DOORBELL to map one bar's inbound address to MSI space. the command COMMAND_DISABLE_DOORBELL to recovery original inbound address mapping. Host side new driver Host side old driver EP: new driver S F EP: old driver F F S: If EP side support MSI, 'pcitest -B' return success. If EP side doesn't support MSI, the same to 'F'. F: 'pcitest -B' return failure, other case as usual. Signed-off-by: Frank Li <Frank.Li@nxp.com> --- Chagne from v4 to v5 - Add doorbell free at unbind function. - Move msi irq handler to here to more complex user case, such as differece doorbell can use difference handler function. - Add Niklas's code to handle fixed bar's case. If need add your signed-off tag or co-developer tag, please let me know. change from v3 to v4 - remove revid requirement - Add command COMMAND_ENABLE_DOORBELL and COMMAND_DISABLE_DOORBELL. - call pci_epc_set_bar() to map inbound address to MSI space only at COMMAND_ENABLE_DOORBELL. --- drivers/pci/endpoint/functions/pci-epf-test.c | 129 ++++++++++++++++++++++++++ 1 file changed, 129 insertions(+)