| Message ID | 20230126213154.1707300-1-tanmay.shah@amd.com (mailing list archive) |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series | [v2] remoteproc: xilinx: add mailbox channels for rpmsg | expand |
Good afternoon, On Thu, Jan 26, 2023 at 01:31:54PM -0800, Tanmay Shah wrote: > This patch makes each r5 core mailbox client and uses > tx and rx channels to send and receive data to/from > remote processor respectively. This is needed for rpmsg > communication to remote processor. > > Signed-off-by: Tanmay Shah <tanmay.shah@amd.com> > --- > > Changes in v2: > - fix vrings carveout names as expeceted by remoteproc framework > This should be in a patch on its own along with a "Fixes" tag. > drivers/remoteproc/xlnx_r5_remoteproc.c | 352 ++++++++++++++++++++---- > 1 file changed, 292 insertions(+), 60 deletions(-) > > diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c > index 2db57d394155..45ce7f2089bf 100644 > --- a/drivers/remoteproc/xlnx_r5_remoteproc.c > +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c > @@ -8,16 +8,23 @@ > #include <linux/dma-mapping.h> > #include <linux/firmware/xlnx-zynqmp.h> > #include <linux/kernel.h> > +#include <linux/mailbox_client.h> > +#include <linux/mailbox/zynqmp-ipi-message.h> > #include <linux/module.h> > #include <linux/of_address.h> > #include <linux/of_platform.h> > #include <linux/of_reserved_mem.h> > #include <linux/platform_device.h> > #include <linux/remoteproc.h> > -#include <linux/slab.h> > > #include "remoteproc_internal.h" > > +/* IPI buffer MAX length */ > +#define IPI_BUF_LEN_MAX 32U > + The documentation for struct zynqmp_ipi_message clearly states that @data is fixed to 12 bytes, whereas here is it set to 32 bytes. Wrong documentation or bug? > +/* RX mailbox client buffer max length */ > +#define MBOX_CLIENT_BUF_MAX (IPI_BUF_LEN_MAX + \ > + sizeof(struct zynqmp_ipi_message)) > /* > * settings for RPU cluster mode which > * reflects possible values of xlnx,cluster-mode dt-property > @@ -65,6 +72,12 @@ static const struct mem_bank_data zynqmp_tcm_banks[] = { > * @rmem: reserved memory region nodes from device tree > * @rproc: rproc handle > * @pm_domain_id: RPU CPU power domain id > + * @rx_mc_buf: to copy data from mailbox rx channel > + * @tx_mc_buf: to copy data to mailbox tx channel > + * @mbox_work: schedule work after receiving data from mailbox > + * @mbox_cl: mailbox client > + * @tx_chan: mailbox tx channel > + * @rx_chan: mailbox rx channel > */ > struct zynqmp_r5_core { > struct device *dev; > @@ -75,6 +88,14 @@ struct zynqmp_r5_core { > struct reserved_mem **rmem; > struct rproc *rproc; > u32 pm_domain_id; > + > + /* mailbox related data structures */ > + unsigned char rx_mc_buf[MBOX_CLIENT_BUF_MAX]; > + unsigned char tx_mc_buf[MBOX_CLIENT_BUF_MAX]; > + struct work_struct mbox_work; > + struct mbox_client mbox_cl; > + struct mbox_chan *tx_chan; > + struct mbox_chan *rx_chan; > }; > > /** > @@ -92,6 +113,181 @@ struct zynqmp_r5_cluster { > struct zynqmp_r5_core **r5_cores; > }; > > +/** > + * event_notified_idr_cb() - callback for vq_interrupt per notifyid > + * @id: rproc->notify id > + * @ptr: pointer to idr private data > + * @data: data passed to idr_for_each callback > + * > + * Pass notification to remoteproc virtio > + * > + * Return: 0. having return is to satisfy the idr_for_each() function > + * pointer input argument requirement. > + **/ > +static int event_notified_idr_cb(int id, void *ptr, void *data) > +{ > + struct rproc *rproc = data; > + > + if (rproc_vq_interrupt(rproc, id) == IRQ_NONE) > + dev_dbg(&rproc->dev, "data not found for vqid=%d\n", id); > + > + return 0; > +} > + > +/** > + * handle_event_notified() - remoteproc notification work function > + * @work: pointer to the work structure > + * > + * It checks each registered remoteproc notify IDs. > + */ > +static void handle_event_notified(struct work_struct *work) > +{ > + struct zynqmp_r5_core *r5_core; > + struct rproc *rproc; > + > + r5_core = container_of(work, struct zynqmp_r5_core, mbox_work); > + rproc = r5_core->rproc; > + > + /* > + * We only use IPI for interrupt. The RPU firmware side may or may > + * not write the notifyid when it trigger IPI. > + * And thus, we scan through all the registered notifyids and > + * find which one is valid to get the message. > + * Even if message from firmware is NULL, we attempt to get vqid > + */ > + idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc); > +} > + > +/** > + * zynqmp_r5_mb_rx_cb() - receive channel mailbox callback > + * @cl: mailbox client > + * @msg: message pointer > + * > + * Receive data from ipi buffer, ack interrupt and then > + * it will schedule the R5 notification work. > + */ > +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *msg) > +{ > + struct zynqmp_ipi_message *ipi_msg, *buf_msg; > + struct zynqmp_r5_core *r5_core; > + size_t len; > + > + r5_core = container_of(cl, struct zynqmp_r5_core, mbox_cl); > + > + /* copy data from ipi buffer to r5_core */ > + ipi_msg = (struct zynqmp_ipi_message *)msg; > + buf_msg = (struct zynqmp_ipi_message *)r5_core->rx_mc_buf; > + len = ipi_msg->len; > + if (len > IPI_BUF_LEN_MAX) { > + dev_warn(r5_core->dev, "msg size exceeded than %d\n", > + IPI_BUF_LEN_MAX); > + len = IPI_BUF_LEN_MAX; > + } > + buf_msg->len = len; > + memcpy(buf_msg->data, ipi_msg->data, len); > + > + /* received and processed interrupt ack */ > + if (mbox_send_message(r5_core->rx_chan, NULL) < 0) > + dev_err(r5_core->dev, "ack failed to mbox rx_chan\n"); > + > + schedule_work(&r5_core->mbox_work); > +} > + > +/** > + * zynqmp_r5_setup_mbox() - Setup mailboxes related properties > + * this is used for each individual R5 core > + * > + * @r5_core: pointer to the ZynqMP r5 core data > + * > + * Function to setup mailboxes related properties > + * > + */ > +static void zynqmp_r5_setup_mbox(struct zynqmp_r5_core *r5_core) > +{ > + struct zynqmp_r5_cluster *cluster; > + struct mbox_client *mbox_cl; > + > + cluster = dev_get_drvdata(r5_core->dev->parent); > + > + /** Extra '*', please remove. > + * ToDo: Use only one IPI channel for APU to communicate with both RPUs > + * in split mode. As of now, two IPI channels are expeceted for APU > + * to communicate with RPU. for example, APU(IPI0)<-> RPU0(IPI1) and > + * APU(IPI7)<->RPU1(IPI2). However, this is not the optimized use > + * of the hardware. As per hardware reference manual, each IPI channel > + * can receive interrupt from another IPI channel. So APU must be able > + * to communicate with both RPUs simultaneously using same IPI channel. > + * For example, this is valid case: APU(IPI0)<->RPU0(IPI1) and > + * APU(IPI0)<->RPU1(IPI2). However, with current available examples > + * and RPU firmware, this configuration in device-tree is causing system-crash. > + * And so, using extra IPI channel is required in device-tree. In split > + * mode explicitly inform user about this limitation and requirement. > + */ > + if (cluster->mode == SPLIT_MODE) > + dev_warn(r5_core->dev, "split mode: APU should use two IPI channels\n"); This comment doesn't do anything useful, please remove. > + > + mbox_cl = &r5_core->mbox_cl; > + mbox_cl->rx_callback = zynqmp_r5_mb_rx_cb; > + mbox_cl->tx_block = false; > + mbox_cl->knows_txdone = false; > + mbox_cl->tx_done = NULL; > + mbox_cl->dev = r5_core->dev; > + > + /* Request TX and RX channels */ > + r5_core->tx_chan = mbox_request_channel_byname(mbox_cl, "tx"); > + if (IS_ERR(r5_core->tx_chan)) { > + r5_core->tx_chan = NULL; > + return; > + } > + > + r5_core->rx_chan = mbox_request_channel_byname(mbox_cl, "rx"); > + if (IS_ERR(r5_core->rx_chan)) { > + mbox_free_channel(r5_core->tx_chan); > + r5_core->rx_chan = NULL; > + r5_core->tx_chan = NULL; > + return; > + } > + > + INIT_WORK(&r5_core->mbox_work, handle_event_notified); > +} > + > +static void zynqmp_r5_free_mbox(struct zynqmp_r5_core *r5_core) > +{ > + if (r5_core->tx_chan) { > + mbox_free_channel(r5_core->tx_chan); > + r5_core->tx_chan = NULL; > + } > + > + if (r5_core->rx_chan) { > + mbox_free_channel(r5_core->rx_chan); > + r5_core->rx_chan = NULL; > + } > +} > + > +/* > + * zynqmp_r5_core_kick() - kick a firmware if mbox is provided > + * @rproc: r5 core's corresponding rproc structure > + * @vqid: virtqueue ID > + */ > +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid) > +{ > + struct zynqmp_r5_core *r5_core = rproc->priv; > + struct device *dev = r5_core->dev; > + struct zynqmp_ipi_message *mb_msg; > + int ret; > + > + /* don't handle kick if mbox setup failed for this core */ > + if (!r5_core->tx_chan && !r5_core->rx_chan) > + return; > + > + mb_msg = (struct zynqmp_ipi_message *)r5_core->tx_mc_buf; > + memcpy(mb_msg->data, &vqid, sizeof(vqid)); > + mb_msg->len = sizeof(vqid); > + ret = mbox_send_message(r5_core->tx_chan, mb_msg); > + if (ret < 0) > + dev_warn(dev, "failed to send message\n"); > +} > + > /* > * zynqmp_r5_set_mode() > * > @@ -227,6 +423,63 @@ static int zynqmp_r5_mem_region_unmap(struct rproc *rproc, > return 0; > } > > +/** > + * zynqmp_r5_get_mem_region_node() > + * parse memory-region property and get reserved mem regions > + * > + * @r5_core: pointer to zynqmp_r5_core type object > + * > + * Return: 0 for success and error code for failure. > + */ > +static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) > +{ > + struct device_node *np, *rmem_np; > + struct reserved_mem **rmem; > + int res_mem_count, i; > + struct device *dev; > + > + dev = r5_core->dev; > + np = r5_core->np; > + > + res_mem_count = of_property_count_elems_of_size(np, "memory-region", > + sizeof(phandle)); > + > + if (res_mem_count <= 0) { > + dev_warn(dev, "failed to get memory-region property %d\n", > + res_mem_count); > + return 0; > + } > + > + if (!r5_core->tx_chan && !r5_core->rx_chan) > + res_mem_count = 1; Hackish, please remove. There should not be a need to mix mailbox information with memory regions. > + > + rmem = devm_kcalloc(dev, res_mem_count, > + sizeof(struct reserved_mem *), GFP_KERNEL); > + if (!rmem) > + return -ENOMEM; > + > + for (i = 0; i < res_mem_count; i++) { > + rmem_np = of_parse_phandle(np, "memory-region", i); > + if (!rmem_np) > + goto release_rmem; > + > + rmem[i] = of_reserved_mem_lookup(rmem_np); > + if (!rmem[i]) { > + of_node_put(rmem_np); > + goto release_rmem; > + } > + > + of_node_put(rmem_np); > + } > + > + r5_core->rmem_count = res_mem_count; > + r5_core->rmem = rmem; > + return 0; > + > +release_rmem: > + return -EINVAL; > +} > + > /* > * add_mem_regions_carveout() > * @rproc: single R5 core's corresponding rproc instance > @@ -241,6 +494,7 @@ static int add_mem_regions_carveout(struct rproc *rproc) > struct zynqmp_r5_core *r5_core; > struct reserved_mem *rmem; > int i, num_mem_regions; > + const char *name; > > r5_core = (struct zynqmp_r5_core *)rproc->priv; > num_mem_regions = r5_core->rmem_count; > @@ -253,15 +507,33 @@ static int add_mem_regions_carveout(struct rproc *rproc) > rproc_mem = rproc_of_resm_mem_entry_init(&rproc->dev, i, > rmem->size, > rmem->base, > - rmem->name); > + "vdev0buffer"); This looks very hackish. > } else { > + /* > + * As per bindings 3rd entry in memory-region property > + * must contain vring0 and 4th entry must contain vring1 > + * memory-regions. For remoteproc framework it is > + * required to have fixed names for these carveouts i.e. > + * in the form of "vdev%dvring%d" where first %d is ID > + * of vdev and second %d is ID of vring. Assign fix names > + * instead of node names, as node names may contain > + * @unit-address as well i.e. vdev0vring0@xxxxxxxx which > + * won't work. > + */ > + if (!strncmp(rmem->name, "vdev0vring0", strlen("vdev0vring0"))) > + name = "vdev0vring0"; > + else if (!strncmp(rmem->name, "vdev0vring1", strlen("vdev0vring1"))) > + name = "vdev0vring1"; > + else > + name = rmem->name; > + So does this. It would be much better to get the right rmem->name before getting to this function, something that should be done in zynqmp_r5_get_mem_region_node(). Look at stm32_rproc_prepare() for an example on how to get the right name reserve memory entries. I am also reasonning this problem has become obvious now that mailboxes are working. That said I also think it should have been caught when the patchset adding support for r5f was worked on. > /* Register associated reserved memory regions */ > rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL, > (dma_addr_t)rmem->base, > rmem->size, rmem->base, > zynqmp_r5_mem_region_map, > zynqmp_r5_mem_region_unmap, > - rmem->name); > + name); > } > > if (!rproc_mem) > @@ -572,6 +844,20 @@ static int zynqmp_r5_rproc_prepare(struct rproc *rproc) > return ret; > } > > + /* > + * If mailbox nodes are disabled using "status" property then setting up > + * mailbox channels will be failed and in that case, we don't need vrings > + * and vdevbuffer for this core. So, setup mailbox before parsing > + * memory-region property. If "tx" and "rx" mailboxes are not setup, then > + * only parse and add first memory-region carveout. As per bindings, it > + * must be firmware load region > + */ > + zynqmp_r5_setup_mbox(rproc->priv); > + Setting up mailboxes should return an error code when not successful. Moreover, why do mailboxes have to be initialised at prepare() time and not once in the probe() function? > + ret = zynqmp_r5_get_mem_region_node(rproc->priv); > + if (ret) > + dev_warn(&rproc->dev, "memory-region prop failed %d\n", ret); > + > ret = add_mem_regions_carveout(rproc); > if (ret) { > dev_err(&rproc->dev, "failed to get reserve mem regions %d\n", ret); > @@ -597,6 +883,8 @@ static int zynqmp_r5_rproc_unprepare(struct rproc *rproc) > > r5_core = (struct zynqmp_r5_core *)rproc->priv; > > + zynqmp_r5_free_mbox(r5_core); > + > for (i = 0; i < r5_core->tcm_bank_count; i++) { > pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id; > if (zynqmp_pm_release_node(pm_domain_id)) > @@ -617,6 +905,7 @@ static const struct rproc_ops zynqmp_r5_rproc_ops = { > .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, > .sanity_check = rproc_elf_sanity_check, > .get_boot_addr = rproc_elf_get_boot_addr, > + .kick = zynqmp_r5_rproc_kick, A kick() function should added only when mailboxes are present rather than invariably as it is now. > }; > > /** > @@ -726,59 +1015,6 @@ static int zynqmp_r5_get_tcm_node(struct zynqmp_r5_cluster *cluster) > return 0; > } > > -/** > - * zynqmp_r5_get_mem_region_node() > - * parse memory-region property and get reserved mem regions > - * > - * @r5_core: pointer to zynqmp_r5_core type object > - * > - * Return: 0 for success and error code for failure. > - */ > -static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) > -{ > - struct device_node *np, *rmem_np; > - struct reserved_mem **rmem; > - int res_mem_count, i; > - struct device *dev; > - > - dev = r5_core->dev; > - np = r5_core->np; > - > - res_mem_count = of_property_count_elems_of_size(np, "memory-region", > - sizeof(phandle)); > - if (res_mem_count <= 0) { > - dev_warn(dev, "failed to get memory-region property %d\n", > - res_mem_count); > - return 0; > - } > - > - rmem = devm_kcalloc(dev, res_mem_count, > - sizeof(struct reserved_mem *), GFP_KERNEL); > - if (!rmem) > - return -ENOMEM; > - > - for (i = 0; i < res_mem_count; i++) { > - rmem_np = of_parse_phandle(np, "memory-region", i); > - if (!rmem_np) > - goto release_rmem; > - > - rmem[i] = of_reserved_mem_lookup(rmem_np); > - if (!rmem[i]) { > - of_node_put(rmem_np); > - goto release_rmem; > - } > - > - of_node_put(rmem_np); > - } > - > - r5_core->rmem_count = res_mem_count; > - r5_core->rmem = rmem; > - return 0; > - > -release_rmem: > - return -EINVAL; > -} > - Why was this moved instead of simply adding a forward declaration at the top of the file? > /* > * zynqmp_r5_core_init() > * Create and initialize zynqmp_r5_core type object > @@ -806,10 +1042,6 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, > for (i = 0; i < cluster->core_count; i++) { > r5_core = cluster->r5_cores[i]; > > - ret = zynqmp_r5_get_mem_region_node(r5_core); > - if (ret) > - dev_warn(dev, "memory-region prop failed %d\n", ret); > - Why doing this since this driver doesn't support attach()/detach() operations yet? Thanks, Mathieu > /* Initialize r5 cores with power-domains parsed from dts */ > ret = of_property_read_u32_index(r5_core->np, "power-domains", > 1, &r5_core->pm_domain_id); > > base-commit: 10de8156ed71d3dbd7e9099aa76e67ea2c37d4ff > -- > 2.25.1 >
Hi Mathieu, Thanks for reviews. Please find my comments below. On 1/31/23 2:59 PM, Mathieu Poirier wrote: > Good afternoon, > > On Thu, Jan 26, 2023 at 01:31:54PM -0800, Tanmay Shah wrote: >> This patch makes each r5 core mailbox client and uses >> tx and rx channels to send and receive data to/from >> remote processor respectively. This is needed for rpmsg >> communication to remote processor. >> >> Signed-off-by: Tanmay Shah <tanmay.shah@amd.com> >> --- >> >> Changes in v2: >> - fix vrings carveout names as expeceted by remoteproc framework >> > This should be in a patch on its own along with a "Fixes" tag. Ack. Next time I will send series of two patches. First patch to fix vrings name with fixes tag and second patch to add mailbox support. > >> drivers/remoteproc/xlnx_r5_remoteproc.c | 352 ++++++++++++++++++++---- >> 1 file changed, 292 insertions(+), 60 deletions(-) >> >> diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c >> index 2db57d394155..45ce7f2089bf 100644 >> --- a/drivers/remoteproc/xlnx_r5_remoteproc.c >> +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c >> @@ -8,16 +8,23 @@ >> #include <linux/dma-mapping.h> >> #include <linux/firmware/xlnx-zynqmp.h> >> #include <linux/kernel.h> >> +#include <linux/mailbox_client.h> >> +#include <linux/mailbox/zynqmp-ipi-message.h> >> #include <linux/module.h> >> #include <linux/of_address.h> >> #include <linux/of_platform.h> >> #include <linux/of_reserved_mem.h> >> #include <linux/platform_device.h> >> #include <linux/remoteproc.h> >> -#include <linux/slab.h> >> >> #include "remoteproc_internal.h" >> >> +/* IPI buffer MAX length */ >> +#define IPI_BUF_LEN_MAX 32U >> + > The documentation for struct zynqmp_ipi_message clearly states that @data is > fixed to 12 bytes, whereas here is it set to 32 bytes. Wrong documentation or > bug? As per hardware reference manual, each message buffer is 32-bytes. There should be bug in IPI driver documentation. I will test this before sending new patch. If 32-bytes are supported, then I will send patch to fix IPI driver with fixes tag as well. >> +/* RX mailbox client buffer max length */ >> +#define MBOX_CLIENT_BUF_MAX (IPI_BUF_LEN_MAX + \ >> + sizeof(struct zynqmp_ipi_message)) >> /* >> * settings for RPU cluster mode which >> * reflects possible values of xlnx,cluster-mode dt-property >> @@ -65,6 +72,12 @@ static const struct mem_bank_data zynqmp_tcm_banks[] = { >> * @rmem: reserved memory region nodes from device tree >> * @rproc: rproc handle >> * @pm_domain_id: RPU CPU power domain id >> + * @rx_mc_buf: to copy data from mailbox rx channel >> + * @tx_mc_buf: to copy data to mailbox tx channel >> + * @mbox_work: schedule work after receiving data from mailbox >> + * @mbox_cl: mailbox client >> + * @tx_chan: mailbox tx channel >> + * @rx_chan: mailbox rx channel >> */ >> struct zynqmp_r5_core { >> struct device *dev; >> @@ -75,6 +88,14 @@ struct zynqmp_r5_core { >> struct reserved_mem **rmem; >> struct rproc *rproc; >> u32 pm_domain_id; >> + >> + /* mailbox related data structures */ >> + unsigned char rx_mc_buf[MBOX_CLIENT_BUF_MAX]; >> + unsigned char tx_mc_buf[MBOX_CLIENT_BUF_MAX]; >> + struct work_struct mbox_work; >> + struct mbox_client mbox_cl; >> + struct mbox_chan *tx_chan; >> + struct mbox_chan *rx_chan; >> }; >> >> /** >> @@ -92,6 +113,181 @@ struct zynqmp_r5_cluster { >> struct zynqmp_r5_core **r5_cores; >> }; >> >> +/** >> + * event_notified_idr_cb() - callback for vq_interrupt per notifyid >> + * @id: rproc->notify id >> + * @ptr: pointer to idr private data >> + * @data: data passed to idr_for_each callback >> + * >> + * Pass notification to remoteproc virtio >> + * >> + * Return: 0. having return is to satisfy the idr_for_each() function >> + * pointer input argument requirement. >> + **/ >> +static int event_notified_idr_cb(int id, void *ptr, void *data) >> +{ >> + struct rproc *rproc = data; >> + >> + if (rproc_vq_interrupt(rproc, id) == IRQ_NONE) >> + dev_dbg(&rproc->dev, "data not found for vqid=%d\n", id); >> + >> + return 0; >> +} >> + >> +/** >> + * handle_event_notified() - remoteproc notification work function >> + * @work: pointer to the work structure >> + * >> + * It checks each registered remoteproc notify IDs. >> + */ >> +static void handle_event_notified(struct work_struct *work) >> +{ >> + struct zynqmp_r5_core *r5_core; >> + struct rproc *rproc; >> + >> + r5_core = container_of(work, struct zynqmp_r5_core, mbox_work); >> + rproc = r5_core->rproc; >> + >> + /* >> + * We only use IPI for interrupt. The RPU firmware side may or may >> + * not write the notifyid when it trigger IPI. >> + * And thus, we scan through all the registered notifyids and >> + * find which one is valid to get the message. >> + * Even if message from firmware is NULL, we attempt to get vqid >> + */ >> + idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc); >> +} >> + >> +/** >> + * zynqmp_r5_mb_rx_cb() - receive channel mailbox callback >> + * @cl: mailbox client >> + * @msg: message pointer >> + * >> + * Receive data from ipi buffer, ack interrupt and then >> + * it will schedule the R5 notification work. >> + */ >> +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *msg) >> +{ >> + struct zynqmp_ipi_message *ipi_msg, *buf_msg; >> + struct zynqmp_r5_core *r5_core; >> + size_t len; >> + >> + r5_core = container_of(cl, struct zynqmp_r5_core, mbox_cl); >> + >> + /* copy data from ipi buffer to r5_core */ >> + ipi_msg = (struct zynqmp_ipi_message *)msg; >> + buf_msg = (struct zynqmp_ipi_message *)r5_core->rx_mc_buf; >> + len = ipi_msg->len; >> + if (len > IPI_BUF_LEN_MAX) { >> + dev_warn(r5_core->dev, "msg size exceeded than %d\n", >> + IPI_BUF_LEN_MAX); >> + len = IPI_BUF_LEN_MAX; >> + } >> + buf_msg->len = len; >> + memcpy(buf_msg->data, ipi_msg->data, len); >> + >> + /* received and processed interrupt ack */ >> + if (mbox_send_message(r5_core->rx_chan, NULL) < 0) >> + dev_err(r5_core->dev, "ack failed to mbox rx_chan\n"); >> + >> + schedule_work(&r5_core->mbox_work); >> +} >> + >> +/** >> + * zynqmp_r5_setup_mbox() - Setup mailboxes related properties >> + * this is used for each individual R5 core >> + * >> + * @r5_core: pointer to the ZynqMP r5 core data >> + * >> + * Function to setup mailboxes related properties >> + * >> + */ >> +static void zynqmp_r5_setup_mbox(struct zynqmp_r5_core *r5_core) >> +{ >> + struct zynqmp_r5_cluster *cluster; >> + struct mbox_client *mbox_cl; >> + >> + cluster = dev_get_drvdata(r5_core->dev->parent); >> + >> + /** > Extra '*', please remove. ACK. > >> + * ToDo: Use only one IPI channel for APU to communicate with both RPUs >> + * in split mode. As of now, two IPI channels are expeceted for APU >> + * to communicate with RPU. for example, APU(IPI0)<-> RPU0(IPI1) and >> + * APU(IPI7)<->RPU1(IPI2). However, this is not the optimized use >> + * of the hardware. As per hardware reference manual, each IPI channel >> + * can receive interrupt from another IPI channel. So APU must be able >> + * to communicate with both RPUs simultaneously using same IPI channel. >> + * For example, this is valid case: APU(IPI0)<->RPU0(IPI1) and >> + * APU(IPI0)<->RPU1(IPI2). However, with current available examples >> + * and RPU firmware, this configuration in device-tree is causing system-crash. >> + * And so, using extra IPI channel is required in device-tree. In split >> + * mode explicitly inform user about this limitation and requirement. >> + */ >> + if (cluster->mode == SPLIT_MODE) >> + dev_warn(r5_core->dev, "split mode: APU should use two IPI channels\n"); > This comment doesn't do anything useful, please remove. ACK. > >> + >> + mbox_cl = &r5_core->mbox_cl; >> + mbox_cl->rx_callback = zynqmp_r5_mb_rx_cb; >> + mbox_cl->tx_block = false; >> + mbox_cl->knows_txdone = false; >> + mbox_cl->tx_done = NULL; >> + mbox_cl->dev = r5_core->dev; >> + >> + /* Request TX and RX channels */ >> + r5_core->tx_chan = mbox_request_channel_byname(mbox_cl, "tx"); >> + if (IS_ERR(r5_core->tx_chan)) { >> + r5_core->tx_chan = NULL; >> + return; >> + } >> + >> + r5_core->rx_chan = mbox_request_channel_byname(mbox_cl, "rx"); >> + if (IS_ERR(r5_core->rx_chan)) { >> + mbox_free_channel(r5_core->tx_chan); >> + r5_core->rx_chan = NULL; >> + r5_core->tx_chan = NULL; >> + return; >> + } >> + >> + INIT_WORK(&r5_core->mbox_work, handle_event_notified); >> +} >> + >> +static void zynqmp_r5_free_mbox(struct zynqmp_r5_core *r5_core) >> +{ >> + if (r5_core->tx_chan) { >> + mbox_free_channel(r5_core->tx_chan); >> + r5_core->tx_chan = NULL; >> + } >> + >> + if (r5_core->rx_chan) { >> + mbox_free_channel(r5_core->rx_chan); >> + r5_core->rx_chan = NULL; >> + } >> +} >> + >> +/* >> + * zynqmp_r5_core_kick() - kick a firmware if mbox is provided >> + * @rproc: r5 core's corresponding rproc structure >> + * @vqid: virtqueue ID >> + */ >> +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid) >> +{ >> + struct zynqmp_r5_core *r5_core = rproc->priv; >> + struct device *dev = r5_core->dev; >> + struct zynqmp_ipi_message *mb_msg; >> + int ret; >> + >> + /* don't handle kick if mbox setup failed for this core */ >> + if (!r5_core->tx_chan && !r5_core->rx_chan) >> + return; >> + >> + mb_msg = (struct zynqmp_ipi_message *)r5_core->tx_mc_buf; >> + memcpy(mb_msg->data, &vqid, sizeof(vqid)); >> + mb_msg->len = sizeof(vqid); >> + ret = mbox_send_message(r5_core->tx_chan, mb_msg); >> + if (ret < 0) >> + dev_warn(dev, "failed to send message\n"); >> +} >> + >> /* >> * zynqmp_r5_set_mode() >> * >> @@ -227,6 +423,63 @@ static int zynqmp_r5_mem_region_unmap(struct rproc *rproc, >> return 0; >> } >> >> +/** >> + * zynqmp_r5_get_mem_region_node() >> + * parse memory-region property and get reserved mem regions >> + * >> + * @r5_core: pointer to zynqmp_r5_core type object >> + * >> + * Return: 0 for success and error code for failure. >> + */ >> +static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) >> +{ >> + struct device_node *np, *rmem_np; >> + struct reserved_mem **rmem; >> + int res_mem_count, i; >> + struct device *dev; >> + >> + dev = r5_core->dev; >> + np = r5_core->np; >> + >> + res_mem_count = of_property_count_elems_of_size(np, "memory-region", >> + sizeof(phandle)); >> + >> + if (res_mem_count <= 0) { >> + dev_warn(dev, "failed to get memory-region property %d\n", >> + res_mem_count); >> + return 0; >> + } >> + >> + if (!r5_core->tx_chan && !r5_core->rx_chan) >> + res_mem_count = 1; > Hackish, please remove. There should not be a need to mix mailbox information > with memory regions. ACK. > >> + >> + rmem = devm_kcalloc(dev, res_mem_count, >> + sizeof(struct reserved_mem *), GFP_KERNEL); >> + if (!rmem) >> + return -ENOMEM; >> + >> + for (i = 0; i < res_mem_count; i++) { >> + rmem_np = of_parse_phandle(np, "memory-region", i); >> + if (!rmem_np) >> + goto release_rmem; >> + >> + rmem[i] = of_reserved_mem_lookup(rmem_np); >> + if (!rmem[i]) { >> + of_node_put(rmem_np); >> + goto release_rmem; >> + } >> + >> + of_node_put(rmem_np); >> + } >> + >> + r5_core->rmem_count = res_mem_count; >> + r5_core->rmem = rmem; >> + return 0; >> + >> +release_rmem: >> + return -EINVAL; >> +} >> + >> /* >> * add_mem_regions_carveout() >> * @rproc: single R5 core's corresponding rproc instance >> @@ -241,6 +494,7 @@ static int add_mem_regions_carveout(struct rproc *rproc) >> struct zynqmp_r5_core *r5_core; >> struct reserved_mem *rmem; >> int i, num_mem_regions; >> + const char *name; >> >> r5_core = (struct zynqmp_r5_core *)rproc->priv; >> num_mem_regions = r5_core->rmem_count; >> @@ -253,15 +507,33 @@ static int add_mem_regions_carveout(struct rproc *rproc) >> rproc_mem = rproc_of_resm_mem_entry_init(&rproc->dev, i, >> rmem->size, >> rmem->base, >> - rmem->name); >> + "vdev0buffer"); > This looks very hackish. > >> } else { >> + /* >> + * As per bindings 3rd entry in memory-region property >> + * must contain vring0 and 4th entry must contain vring1 >> + * memory-regions. For remoteproc framework it is >> + * required to have fixed names for these carveouts i.e. >> + * in the form of "vdev%dvring%d" where first %d is ID >> + * of vdev and second %d is ID of vring. Assign fix names >> + * instead of node names, as node names may contain >> + * @unit-address as well i.e. vdev0vring0@xxxxxxxx which >> + * won't work. >> + */ >> + if (!strncmp(rmem->name, "vdev0vring0", strlen("vdev0vring0"))) >> + name = "vdev0vring0"; >> + else if (!strncmp(rmem->name, "vdev0vring1", strlen("vdev0vring1"))) >> + name = "vdev0vring1"; >> + else >> + name = rmem->name; >> + > So does this. It would be much better to get the right rmem->name before > getting to this function, something that should be done in > zynqmp_r5_get_mem_region_node(). Look at stm32_rproc_prepare() for an example > on how to get the right name reserve memory entries. > > I am also reasonning this problem has become obvious now that mailboxes are > working. That said I also think it should have been caught when the patchset > adding support for r5f was worked on. Yes correct. Actually I had tested with only one core at a time. During testing my device-tree had only node name and not node address (i.e. @xxxxxx) So, I couldn't catch the issue. Also, mailbox wasn't supported so I couldn't put vrings to use at that time. I started facing the issue when both cores were up simultaneously and I couldn't put duplicate node names and I started adding vrings node addresses. I will see how to parse node-name only from format "node-name@unit-address". That should resolve all above hacks. Thanks, Tanmay >> /* Register associated reserved memory regions */ >> rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL, >> (dma_addr_t)rmem->base, >> rmem->size, rmem->base, >> zynqmp_r5_mem_region_map, >> zynqmp_r5_mem_region_unmap, >> - rmem->name); >> + name); >> } >> >> if (!rproc_mem) >> @@ -572,6 +844,20 @@ static int zynqmp_r5_rproc_prepare(struct rproc *rproc) >> return ret; >> } >> >> + /* >> + * If mailbox nodes are disabled using "status" property then setting up >> + * mailbox channels will be failed and in that case, we don't need vrings >> + * and vdevbuffer for this core. So, setup mailbox before parsing >> + * memory-region property. If "tx" and "rx" mailboxes are not setup, then >> + * only parse and add first memory-region carveout. As per bindings, it >> + * must be firmware load region >> + */ >> + zynqmp_r5_setup_mbox(rproc->priv); >> + > Setting up mailboxes should return an error code when not successful. In case of failure that function is printing relative error messages. However, If mailbox nodes are disabled in device-tree with status property, then it is expected that mailbox setup will fail. However, that should not stop remoteproc LCM functionality as user still should be able to start/stop/loadfw functionality. So, I did not see need to return error code. I will add return error code, however that won't stop rest of the driver functionality. I can just print some warning message based on error code. > Moreover, > why do mailboxes have to be initialised at prepare() time and not once in the > probe() function? ACK. This part I will take care. I will let you know if I face any problems while moving this to probe otherwise I am okay. > >> + ret = zynqmp_r5_get_mem_region_node(rproc->priv); >> + if (ret) >> + dev_warn(&rproc->dev, "memory-region prop failed %d\n", ret); >> + >> ret = add_mem_regions_carveout(rproc); >> if (ret) { >> dev_err(&rproc->dev, "failed to get reserve mem regions %d\n", ret); >> @@ -597,6 +883,8 @@ static int zynqmp_r5_rproc_unprepare(struct rproc *rproc) >> >> r5_core = (struct zynqmp_r5_core *)rproc->priv; >> >> + zynqmp_r5_free_mbox(r5_core); >> + >> for (i = 0; i < r5_core->tcm_bank_count; i++) { >> pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id; >> if (zynqmp_pm_release_node(pm_domain_id)) >> @@ -617,6 +905,7 @@ static const struct rproc_ops zynqmp_r5_rproc_ops = { >> .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, >> .sanity_check = rproc_elf_sanity_check, >> .get_boot_addr = rproc_elf_get_boot_addr, >> + .kick = zynqmp_r5_rproc_kick, > A kick() function should added only when mailboxes are present rather than > invariably as it is now. May be I am missing something but, I believe this is const variable and I may not be able to modify it once initialized. Is it ok to remove const? then I can take care of adding kick based on mbox is available or not. > >> }; >> >> /** >> @@ -726,59 +1015,6 @@ static int zynqmp_r5_get_tcm_node(struct zynqmp_r5_cluster *cluster) >> return 0; >> } >> >> -/** >> - * zynqmp_r5_get_mem_region_node() >> - * parse memory-region property and get reserved mem regions >> - * >> - * @r5_core: pointer to zynqmp_r5_core type object >> - * >> - * Return: 0 for success and error code for failure. >> - */ >> -static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) >> -{ >> - struct device_node *np, *rmem_np; >> - struct reserved_mem **rmem; >> - int res_mem_count, i; >> - struct device *dev; >> - >> - dev = r5_core->dev; >> - np = r5_core->np; >> - >> - res_mem_count = of_property_count_elems_of_size(np, "memory-region", >> - sizeof(phandle)); >> - if (res_mem_count <= 0) { >> - dev_warn(dev, "failed to get memory-region property %d\n", >> - res_mem_count); >> - return 0; >> - } >> - >> - rmem = devm_kcalloc(dev, res_mem_count, >> - sizeof(struct reserved_mem *), GFP_KERNEL); >> - if (!rmem) >> - return -ENOMEM; >> - >> - for (i = 0; i < res_mem_count; i++) { >> - rmem_np = of_parse_phandle(np, "memory-region", i); >> - if (!rmem_np) >> - goto release_rmem; >> - >> - rmem[i] = of_reserved_mem_lookup(rmem_np); >> - if (!rmem[i]) { >> - of_node_put(rmem_np); >> - goto release_rmem; >> - } >> - >> - of_node_put(rmem_np); >> - } >> - >> - r5_core->rmem_count = res_mem_count; >> - r5_core->rmem = rmem; >> - return 0; >> - >> -release_rmem: >> - return -EINVAL; >> -} >> - > Why was this moved instead of simply adding a forward declaration at the top of > the file? ACK. That's good idea. Thanks! > >> /* >> * zynqmp_r5_core_init() >> * Create and initialize zynqmp_r5_core type object >> @@ -806,10 +1042,6 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, >> for (i = 0; i < cluster->core_count; i++) { >> r5_core = cluster->r5_cores[i]; >> >> - ret = zynqmp_r5_get_mem_region_node(r5_core); >> - if (ret) >> - dev_warn(dev, "memory-region prop failed %d\n", ret); >> - > Why doing this since this driver doesn't support attach()/detach() operations > yet? I see, so we should always fail if memory-region property isn't defined? Actually it is also possible to load and boot firmware completely out of TCM. Since the driver has TCM addresses, we don't really need memory-region property at all in that case. So by not failing when memory-region is not defined, we are giving chance to load and boot firmware from TCM. I can add this in comment. Thanks, Tanmay > Thanks, > Mathieu > >> /* Initialize r5 cores with power-domains parsed from dts */ >> ret = of_property_read_u32_index(r5_core->np, "power-domains", >> 1, &r5_core->pm_domain_id); >> >> base-commit: 10de8156ed71d3dbd7e9099aa76e67ea2c37d4ff >> -- >> 2.25.1 >>
On Thu, Feb 02, 2023 at 10:17:28AM -0800, Tanmay Shah wrote: > Hi Mathieu, > > Thanks for reviews. > > Please find my comments below. > > On 1/31/23 2:59 PM, Mathieu Poirier wrote: > > Good afternoon, > > > > On Thu, Jan 26, 2023 at 01:31:54PM -0800, Tanmay Shah wrote: > > > This patch makes each r5 core mailbox client and uses > > > tx and rx channels to send and receive data to/from > > > remote processor respectively. This is needed for rpmsg > > > communication to remote processor. > > > > > > Signed-off-by: Tanmay Shah <tanmay.shah@amd.com> > > > --- > > > > > > Changes in v2: > > > - fix vrings carveout names as expeceted by remoteproc framework > > > > > This should be in a patch on its own along with a "Fixes" tag. > > Ack. > > Next time I will send series of two patches. > > First patch to fix vrings name with fixes tag and second patch to add > mailbox support. > > > > > > > drivers/remoteproc/xlnx_r5_remoteproc.c | 352 ++++++++++++++++++++---- > > > 1 file changed, 292 insertions(+), 60 deletions(-) > > > > > > diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c > > > index 2db57d394155..45ce7f2089bf 100644 > > > --- a/drivers/remoteproc/xlnx_r5_remoteproc.c > > > +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c > > > @@ -8,16 +8,23 @@ > > > #include <linux/dma-mapping.h> > > > #include <linux/firmware/xlnx-zynqmp.h> > > > #include <linux/kernel.h> > > > +#include <linux/mailbox_client.h> > > > +#include <linux/mailbox/zynqmp-ipi-message.h> > > > #include <linux/module.h> > > > #include <linux/of_address.h> > > > #include <linux/of_platform.h> > > > #include <linux/of_reserved_mem.h> > > > #include <linux/platform_device.h> > > > #include <linux/remoteproc.h> > > > -#include <linux/slab.h> > > > #include "remoteproc_internal.h" > > > +/* IPI buffer MAX length */ > > > +#define IPI_BUF_LEN_MAX 32U > > > + > > The documentation for struct zynqmp_ipi_message clearly states that @data is > > fixed to 12 bytes, whereas here is it set to 32 bytes. Wrong documentation or > > bug? > > As per hardware reference manual, each message buffer is 32-bytes. There > should be bug in IPI driver documentation. > > I will test this before sending new patch. If 32-bytes are supported, then I > will send patch to fix IPI driver with fixes tag as well. > > > > > +/* RX mailbox client buffer max length */ > > > +#define MBOX_CLIENT_BUF_MAX (IPI_BUF_LEN_MAX + \ > > > + sizeof(struct zynqmp_ipi_message)) > > > /* > > > * settings for RPU cluster mode which > > > * reflects possible values of xlnx,cluster-mode dt-property > > > @@ -65,6 +72,12 @@ static const struct mem_bank_data zynqmp_tcm_banks[] = { > > > * @rmem: reserved memory region nodes from device tree > > > * @rproc: rproc handle > > > * @pm_domain_id: RPU CPU power domain id > > > + * @rx_mc_buf: to copy data from mailbox rx channel > > > + * @tx_mc_buf: to copy data to mailbox tx channel > > > + * @mbox_work: schedule work after receiving data from mailbox > > > + * @mbox_cl: mailbox client > > > + * @tx_chan: mailbox tx channel > > > + * @rx_chan: mailbox rx channel > > > */ > > > struct zynqmp_r5_core { > > > struct device *dev; > > > @@ -75,6 +88,14 @@ struct zynqmp_r5_core { > > > struct reserved_mem **rmem; > > > struct rproc *rproc; > > > u32 pm_domain_id; > > > + > > > + /* mailbox related data structures */ > > > + unsigned char rx_mc_buf[MBOX_CLIENT_BUF_MAX]; > > > + unsigned char tx_mc_buf[MBOX_CLIENT_BUF_MAX]; > > > + struct work_struct mbox_work; > > > + struct mbox_client mbox_cl; > > > + struct mbox_chan *tx_chan; > > > + struct mbox_chan *rx_chan; > > > }; > > > /** > > > @@ -92,6 +113,181 @@ struct zynqmp_r5_cluster { > > > struct zynqmp_r5_core **r5_cores; > > > }; > > > +/** > > > + * event_notified_idr_cb() - callback for vq_interrupt per notifyid > > > + * @id: rproc->notify id > > > + * @ptr: pointer to idr private data > > > + * @data: data passed to idr_for_each callback > > > + * > > > + * Pass notification to remoteproc virtio > > > + * > > > + * Return: 0. having return is to satisfy the idr_for_each() function > > > + * pointer input argument requirement. > > > + **/ > > > +static int event_notified_idr_cb(int id, void *ptr, void *data) > > > +{ > > > + struct rproc *rproc = data; > > > + > > > + if (rproc_vq_interrupt(rproc, id) == IRQ_NONE) > > > + dev_dbg(&rproc->dev, "data not found for vqid=%d\n", id); > > > + > > > + return 0; > > > +} > > > + > > > +/** > > > + * handle_event_notified() - remoteproc notification work function > > > + * @work: pointer to the work structure > > > + * > > > + * It checks each registered remoteproc notify IDs. > > > + */ > > > +static void handle_event_notified(struct work_struct *work) > > > +{ > > > + struct zynqmp_r5_core *r5_core; > > > + struct rproc *rproc; > > > + > > > + r5_core = container_of(work, struct zynqmp_r5_core, mbox_work); > > > + rproc = r5_core->rproc; > > > + > > > + /* > > > + * We only use IPI for interrupt. The RPU firmware side may or may > > > + * not write the notifyid when it trigger IPI. > > > + * And thus, we scan through all the registered notifyids and > > > + * find which one is valid to get the message. > > > + * Even if message from firmware is NULL, we attempt to get vqid > > > + */ > > > + idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc); > > > +} > > > + > > > +/** > > > + * zynqmp_r5_mb_rx_cb() - receive channel mailbox callback > > > + * @cl: mailbox client > > > + * @msg: message pointer > > > + * > > > + * Receive data from ipi buffer, ack interrupt and then > > > + * it will schedule the R5 notification work. > > > + */ > > > +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *msg) > > > +{ > > > + struct zynqmp_ipi_message *ipi_msg, *buf_msg; > > > + struct zynqmp_r5_core *r5_core; > > > + size_t len; > > > + > > > + r5_core = container_of(cl, struct zynqmp_r5_core, mbox_cl); > > > + > > > + /* copy data from ipi buffer to r5_core */ > > > + ipi_msg = (struct zynqmp_ipi_message *)msg; > > > + buf_msg = (struct zynqmp_ipi_message *)r5_core->rx_mc_buf; > > > + len = ipi_msg->len; > > > + if (len > IPI_BUF_LEN_MAX) { > > > + dev_warn(r5_core->dev, "msg size exceeded than %d\n", > > > + IPI_BUF_LEN_MAX); > > > + len = IPI_BUF_LEN_MAX; > > > + } > > > + buf_msg->len = len; > > > + memcpy(buf_msg->data, ipi_msg->data, len); > > > + > > > + /* received and processed interrupt ack */ > > > + if (mbox_send_message(r5_core->rx_chan, NULL) < 0) > > > + dev_err(r5_core->dev, "ack failed to mbox rx_chan\n"); > > > + > > > + schedule_work(&r5_core->mbox_work); > > > +} > > > + > > > +/** > > > + * zynqmp_r5_setup_mbox() - Setup mailboxes related properties > > > + * this is used for each individual R5 core > > > + * > > > + * @r5_core: pointer to the ZynqMP r5 core data > > > + * > > > + * Function to setup mailboxes related properties > > > + * > > > + */ > > > +static void zynqmp_r5_setup_mbox(struct zynqmp_r5_core *r5_core) > > > +{ > > > + struct zynqmp_r5_cluster *cluster; > > > + struct mbox_client *mbox_cl; > > > + > > > + cluster = dev_get_drvdata(r5_core->dev->parent); > > > + > > > + /** > > Extra '*', please remove. > > ACK. > > > > > > > + * ToDo: Use only one IPI channel for APU to communicate with both RPUs > > > + * in split mode. As of now, two IPI channels are expeceted for APU > > > + * to communicate with RPU. for example, APU(IPI0)<-> RPU0(IPI1) and > > > + * APU(IPI7)<->RPU1(IPI2). However, this is not the optimized use > > > + * of the hardware. As per hardware reference manual, each IPI channel > > > + * can receive interrupt from another IPI channel. So APU must be able > > > + * to communicate with both RPUs simultaneously using same IPI channel. > > > + * For example, this is valid case: APU(IPI0)<->RPU0(IPI1) and > > > + * APU(IPI0)<->RPU1(IPI2). However, with current available examples > > > + * and RPU firmware, this configuration in device-tree is causing system-crash. > > > + * And so, using extra IPI channel is required in device-tree. In split > > > + * mode explicitly inform user about this limitation and requirement. > > > + */ > > > + if (cluster->mode == SPLIT_MODE) > > > + dev_warn(r5_core->dev, "split mode: APU should use two IPI channels\n"); > > This comment doesn't do anything useful, please remove. > > > ACK. > > > > > > > + > > > + mbox_cl = &r5_core->mbox_cl; > > > + mbox_cl->rx_callback = zynqmp_r5_mb_rx_cb; > > > + mbox_cl->tx_block = false; > > > + mbox_cl->knows_txdone = false; > > > + mbox_cl->tx_done = NULL; > > > + mbox_cl->dev = r5_core->dev; > > > + > > > + /* Request TX and RX channels */ > > > + r5_core->tx_chan = mbox_request_channel_byname(mbox_cl, "tx"); > > > + if (IS_ERR(r5_core->tx_chan)) { > > > + r5_core->tx_chan = NULL; > > > + return; > > > + } > > > + > > > + r5_core->rx_chan = mbox_request_channel_byname(mbox_cl, "rx"); > > > + if (IS_ERR(r5_core->rx_chan)) { > > > + mbox_free_channel(r5_core->tx_chan); > > > + r5_core->rx_chan = NULL; > > > + r5_core->tx_chan = NULL; > > > + return; > > > + } > > > + > > > + INIT_WORK(&r5_core->mbox_work, handle_event_notified); > > > +} > > > + > > > +static void zynqmp_r5_free_mbox(struct zynqmp_r5_core *r5_core) > > > +{ > > > + if (r5_core->tx_chan) { > > > + mbox_free_channel(r5_core->tx_chan); > > > + r5_core->tx_chan = NULL; > > > + } > > > + > > > + if (r5_core->rx_chan) { > > > + mbox_free_channel(r5_core->rx_chan); > > > + r5_core->rx_chan = NULL; > > > + } > > > +} > > > + > > > +/* > > > + * zynqmp_r5_core_kick() - kick a firmware if mbox is provided > > > + * @rproc: r5 core's corresponding rproc structure > > > + * @vqid: virtqueue ID > > > + */ > > > +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid) > > > +{ > > > + struct zynqmp_r5_core *r5_core = rproc->priv; > > > + struct device *dev = r5_core->dev; > > > + struct zynqmp_ipi_message *mb_msg; > > > + int ret; > > > + > > > + /* don't handle kick if mbox setup failed for this core */ > > > + if (!r5_core->tx_chan && !r5_core->rx_chan) > > > + return; > > > + > > > + mb_msg = (struct zynqmp_ipi_message *)r5_core->tx_mc_buf; > > > + memcpy(mb_msg->data, &vqid, sizeof(vqid)); > > > + mb_msg->len = sizeof(vqid); > > > + ret = mbox_send_message(r5_core->tx_chan, mb_msg); > > > + if (ret < 0) > > > + dev_warn(dev, "failed to send message\n"); > > > +} > > > + > > > /* > > > * zynqmp_r5_set_mode() > > > * > > > @@ -227,6 +423,63 @@ static int zynqmp_r5_mem_region_unmap(struct rproc *rproc, > > > return 0; > > > } > > > +/** > > > + * zynqmp_r5_get_mem_region_node() > > > + * parse memory-region property and get reserved mem regions > > > + * > > > + * @r5_core: pointer to zynqmp_r5_core type object > > > + * > > > + * Return: 0 for success and error code for failure. > > > + */ > > > +static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) > > > +{ > > > + struct device_node *np, *rmem_np; > > > + struct reserved_mem **rmem; > > > + int res_mem_count, i; > > > + struct device *dev; > > > + > > > + dev = r5_core->dev; > > > + np = r5_core->np; > > > + > > > + res_mem_count = of_property_count_elems_of_size(np, "memory-region", > > > + sizeof(phandle)); > > > + > > > + if (res_mem_count <= 0) { > > > + dev_warn(dev, "failed to get memory-region property %d\n", > > > + res_mem_count); > > > + return 0; > > > + } > > > + > > > + if (!r5_core->tx_chan && !r5_core->rx_chan) > > > + res_mem_count = 1; > > Hackish, please remove. There should not be a need to mix mailbox information > > with memory regions. > > ACK. > > > > > > > + > > > + rmem = devm_kcalloc(dev, res_mem_count, > > > + sizeof(struct reserved_mem *), GFP_KERNEL); > > > + if (!rmem) > > > + return -ENOMEM; > > > + > > > + for (i = 0; i < res_mem_count; i++) { > > > + rmem_np = of_parse_phandle(np, "memory-region", i); > > > + if (!rmem_np) > > > + goto release_rmem; > > > + > > > + rmem[i] = of_reserved_mem_lookup(rmem_np); > > > + if (!rmem[i]) { > > > + of_node_put(rmem_np); > > > + goto release_rmem; > > > + } > > > + > > > + of_node_put(rmem_np); > > > + } > > > + > > > + r5_core->rmem_count = res_mem_count; > > > + r5_core->rmem = rmem; > > > + return 0; > > > + > > > +release_rmem: > > > + return -EINVAL; > > > +} > > > + > > > /* > > > * add_mem_regions_carveout() > > > * @rproc: single R5 core's corresponding rproc instance > > > @@ -241,6 +494,7 @@ static int add_mem_regions_carveout(struct rproc *rproc) > > > struct zynqmp_r5_core *r5_core; > > > struct reserved_mem *rmem; > > > int i, num_mem_regions; > > > + const char *name; > > > r5_core = (struct zynqmp_r5_core *)rproc->priv; > > > num_mem_regions = r5_core->rmem_count; > > > @@ -253,15 +507,33 @@ static int add_mem_regions_carveout(struct rproc *rproc) > > > rproc_mem = rproc_of_resm_mem_entry_init(&rproc->dev, i, > > > rmem->size, > > > rmem->base, > > > - rmem->name); > > > + "vdev0buffer"); > > This looks very hackish. > > > > > } else { > > > + /* > > > + * As per bindings 3rd entry in memory-region property > > > + * must contain vring0 and 4th entry must contain vring1 > > > + * memory-regions. For remoteproc framework it is > > > + * required to have fixed names for these carveouts i.e. > > > + * in the form of "vdev%dvring%d" where first %d is ID > > > + * of vdev and second %d is ID of vring. Assign fix names > > > + * instead of node names, as node names may contain > > > + * @unit-address as well i.e. vdev0vring0@xxxxxxxx which > > > + * won't work. > > > + */ > > > + if (!strncmp(rmem->name, "vdev0vring0", strlen("vdev0vring0"))) > > > + name = "vdev0vring0"; > > > + else if (!strncmp(rmem->name, "vdev0vring1", strlen("vdev0vring1"))) > > > + name = "vdev0vring1"; > > > + else > > > + name = rmem->name; > > > + > > So does this. It would be much better to get the right rmem->name before > > getting to this function, something that should be done in > > zynqmp_r5_get_mem_region_node(). Look at stm32_rproc_prepare() for an example > > on how to get the right name reserve memory entries. > > > > I am also reasonning this problem has become obvious now that mailboxes are > > working. That said I also think it should have been caught when the patchset > > adding support for r5f was worked on. > > Yes correct. Actually I had tested with only one core at a time. > > During testing my device-tree had only node name and not node address (i.e. > @xxxxxx) > > So, I couldn't catch the issue. Also, mailbox wasn't supported so I couldn't > put vrings to use at > > that time. I started facing the issue when both cores were up simultaneously > and I couldn't put > > duplicate node names and I started adding vrings node addresses. > > I will see how to parse node-name only from format "node-name@unit-address". > > That should resolve all above hacks. > > Thanks, > > Tanmay > > > > > /* Register associated reserved memory regions */ > > > rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL, > > > (dma_addr_t)rmem->base, > > > rmem->size, rmem->base, > > > zynqmp_r5_mem_region_map, > > > zynqmp_r5_mem_region_unmap, > > > - rmem->name); > > > + name); > > > } > > > if (!rproc_mem) > > > @@ -572,6 +844,20 @@ static int zynqmp_r5_rproc_prepare(struct rproc *rproc) > > > return ret; > > > } > > > + /* > > > + * If mailbox nodes are disabled using "status" property then setting up > > > + * mailbox channels will be failed and in that case, we don't need vrings > > > + * and vdevbuffer for this core. So, setup mailbox before parsing > > > + * memory-region property. If "tx" and "rx" mailboxes are not setup, then > > > + * only parse and add first memory-region carveout. As per bindings, it > > > + * must be firmware load region > > > + */ > > > + zynqmp_r5_setup_mbox(rproc->priv); > > > + > > Setting up mailboxes should return an error code when not successful. > > > In case of failure that function is printing relative error messages. > However, If mailbox nodes > > are disabled in device-tree with status property, then it is expected that > mailbox setup will fail. However, > > that should not stop remoteproc LCM functionality as user still should be > able to start/stop/loadfw functionality. > > So, I did not see need to return error code. > > I will add return error code, however that won't stop rest of the driver > functionality. I can just print some warning > > message based on error code. > > > > Moreover, > > why do mailboxes have to be initialised at prepare() time and not once in the > > probe() function? > > ACK. > > This part I will take care. I will let you know if I face any problems while > moving this to probe otherwise I am okay. > > > > > > > + ret = zynqmp_r5_get_mem_region_node(rproc->priv); > > > + if (ret) > > > + dev_warn(&rproc->dev, "memory-region prop failed %d\n", ret); > > > + > > > ret = add_mem_regions_carveout(rproc); > > > if (ret) { > > > dev_err(&rproc->dev, "failed to get reserve mem regions %d\n", ret); > > > @@ -597,6 +883,8 @@ static int zynqmp_r5_rproc_unprepare(struct rproc *rproc) > > > r5_core = (struct zynqmp_r5_core *)rproc->priv; > > > + zynqmp_r5_free_mbox(r5_core); > > > + > > > for (i = 0; i < r5_core->tcm_bank_count; i++) { > > > pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id; > > > if (zynqmp_pm_release_node(pm_domain_id)) > > > @@ -617,6 +905,7 @@ static const struct rproc_ops zynqmp_r5_rproc_ops = { > > > .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, > > > .sanity_check = rproc_elf_sanity_check, > > > .get_boot_addr = rproc_elf_get_boot_addr, > > > + .kick = zynqmp_r5_rproc_kick, > > A kick() function should added only when mailboxes are present rather than > > invariably as it is now. > > > May be I am missing something but, I believe this is const variable and I > may not be able to modify it once initialized. > > Is it ok to remove const? then I can take care of adding kick based on mbox > is available or not. > Exactly. Yes, the "const" can be removed. > > > > > > }; > > > /** > > > @@ -726,59 +1015,6 @@ static int zynqmp_r5_get_tcm_node(struct zynqmp_r5_cluster *cluster) > > > return 0; > > > } > > > -/** > > > - * zynqmp_r5_get_mem_region_node() > > > - * parse memory-region property and get reserved mem regions > > > - * > > > - * @r5_core: pointer to zynqmp_r5_core type object > > > - * > > > - * Return: 0 for success and error code for failure. > > > - */ > > > -static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) > > > -{ > > > - struct device_node *np, *rmem_np; > > > - struct reserved_mem **rmem; > > > - int res_mem_count, i; > > > - struct device *dev; > > > - > > > - dev = r5_core->dev; > > > - np = r5_core->np; > > > - > > > - res_mem_count = of_property_count_elems_of_size(np, "memory-region", > > > - sizeof(phandle)); > > > - if (res_mem_count <= 0) { > > > - dev_warn(dev, "failed to get memory-region property %d\n", > > > - res_mem_count); > > > - return 0; > > > - } > > > - > > > - rmem = devm_kcalloc(dev, res_mem_count, > > > - sizeof(struct reserved_mem *), GFP_KERNEL); > > > - if (!rmem) > > > - return -ENOMEM; > > > - > > > - for (i = 0; i < res_mem_count; i++) { > > > - rmem_np = of_parse_phandle(np, "memory-region", i); > > > - if (!rmem_np) > > > - goto release_rmem; > > > - > > > - rmem[i] = of_reserved_mem_lookup(rmem_np); > > > - if (!rmem[i]) { > > > - of_node_put(rmem_np); > > > - goto release_rmem; > > > - } > > > - > > > - of_node_put(rmem_np); > > > - } > > > - > > > - r5_core->rmem_count = res_mem_count; > > > - r5_core->rmem = rmem; > > > - return 0; > > > - > > > -release_rmem: > > > - return -EINVAL; > > > -} > > > - > > Why was this moved instead of simply adding a forward declaration at the top of > > the file? > > > ACK. That's good idea. Thanks! > > > > > > > /* > > > * zynqmp_r5_core_init() > > > * Create and initialize zynqmp_r5_core type object > > > @@ -806,10 +1042,6 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, > > > for (i = 0; i < cluster->core_count; i++) { > > > r5_core = cluster->r5_cores[i]; > > > - ret = zynqmp_r5_get_mem_region_node(r5_core); > > > - if (ret) > > > - dev_warn(dev, "memory-region prop failed %d\n", ret); > > > - > > Why doing this since this driver doesn't support attach()/detach() operations > > yet? > > > I see, so we should always fail if memory-region property isn't defined? My comment is about moving zynqmp_r5_get_mem_region_node() to the prepare() function. There shouldn't be a need to do that. > > Actually it is also possible to load and boot firmware completely out of > TCM. > > Since the driver has TCM addresses, we don't really need memory-region > property at all in > > that case. So by not failing when memory-region is not defined, we are > giving chance to > > load and boot firmware from TCM. I can add this in comment. > > Thanks, > > Tanmay > > > > Thanks, > > Mathieu > > > > > /* Initialize r5 cores with power-domains parsed from dts */ > > > ret = of_property_read_u32_index(r5_core->np, "power-domains", > > > 1, &r5_core->pm_domain_id); > > > > > > base-commit: 10de8156ed71d3dbd7e9099aa76e67ea2c37d4ff > > > -- > > > 2.25.1 > > >
diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c index 2db57d394155..45ce7f2089bf 100644 --- a/drivers/remoteproc/xlnx_r5_remoteproc.c +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c @@ -8,16 +8,23 @@ #include <linux/dma-mapping.h> #include <linux/firmware/xlnx-zynqmp.h> #include <linux/kernel.h> +#include <linux/mailbox_client.h> +#include <linux/mailbox/zynqmp-ipi-message.h> #include <linux/module.h> #include <linux/of_address.h> #include <linux/of_platform.h> #include <linux/of_reserved_mem.h> #include <linux/platform_device.h> #include <linux/remoteproc.h> -#include <linux/slab.h> #include "remoteproc_internal.h" +/* IPI buffer MAX length */ +#define IPI_BUF_LEN_MAX 32U + +/* RX mailbox client buffer max length */ +#define MBOX_CLIENT_BUF_MAX (IPI_BUF_LEN_MAX + \ + sizeof(struct zynqmp_ipi_message)) /* * settings for RPU cluster mode which * reflects possible values of xlnx,cluster-mode dt-property @@ -65,6 +72,12 @@ static const struct mem_bank_data zynqmp_tcm_banks[] = { * @rmem: reserved memory region nodes from device tree * @rproc: rproc handle * @pm_domain_id: RPU CPU power domain id + * @rx_mc_buf: to copy data from mailbox rx channel + * @tx_mc_buf: to copy data to mailbox tx channel + * @mbox_work: schedule work after receiving data from mailbox + * @mbox_cl: mailbox client + * @tx_chan: mailbox tx channel + * @rx_chan: mailbox rx channel */ struct zynqmp_r5_core { struct device *dev; @@ -75,6 +88,14 @@ struct zynqmp_r5_core { struct reserved_mem **rmem; struct rproc *rproc; u32 pm_domain_id; + + /* mailbox related data structures */ + unsigned char rx_mc_buf[MBOX_CLIENT_BUF_MAX]; + unsigned char tx_mc_buf[MBOX_CLIENT_BUF_MAX]; + struct work_struct mbox_work; + struct mbox_client mbox_cl; + struct mbox_chan *tx_chan; + struct mbox_chan *rx_chan; }; /** @@ -92,6 +113,181 @@ struct zynqmp_r5_cluster { struct zynqmp_r5_core **r5_cores; }; +/** + * event_notified_idr_cb() - callback for vq_interrupt per notifyid + * @id: rproc->notify id + * @ptr: pointer to idr private data + * @data: data passed to idr_for_each callback + * + * Pass notification to remoteproc virtio + * + * Return: 0. having return is to satisfy the idr_for_each() function + * pointer input argument requirement. + **/ +static int event_notified_idr_cb(int id, void *ptr, void *data) +{ + struct rproc *rproc = data; + + if (rproc_vq_interrupt(rproc, id) == IRQ_NONE) + dev_dbg(&rproc->dev, "data not found for vqid=%d\n", id); + + return 0; +} + +/** + * handle_event_notified() - remoteproc notification work function + * @work: pointer to the work structure + * + * It checks each registered remoteproc notify IDs. + */ +static void handle_event_notified(struct work_struct *work) +{ + struct zynqmp_r5_core *r5_core; + struct rproc *rproc; + + r5_core = container_of(work, struct zynqmp_r5_core, mbox_work); + rproc = r5_core->rproc; + + /* + * We only use IPI for interrupt. The RPU firmware side may or may + * not write the notifyid when it trigger IPI. + * And thus, we scan through all the registered notifyids and + * find which one is valid to get the message. + * Even if message from firmware is NULL, we attempt to get vqid + */ + idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc); +} + +/** + * zynqmp_r5_mb_rx_cb() - receive channel mailbox callback + * @cl: mailbox client + * @msg: message pointer + * + * Receive data from ipi buffer, ack interrupt and then + * it will schedule the R5 notification work. + */ +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *msg) +{ + struct zynqmp_ipi_message *ipi_msg, *buf_msg; + struct zynqmp_r5_core *r5_core; + size_t len; + + r5_core = container_of(cl, struct zynqmp_r5_core, mbox_cl); + + /* copy data from ipi buffer to r5_core */ + ipi_msg = (struct zynqmp_ipi_message *)msg; + buf_msg = (struct zynqmp_ipi_message *)r5_core->rx_mc_buf; + len = ipi_msg->len; + if (len > IPI_BUF_LEN_MAX) { + dev_warn(r5_core->dev, "msg size exceeded than %d\n", + IPI_BUF_LEN_MAX); + len = IPI_BUF_LEN_MAX; + } + buf_msg->len = len; + memcpy(buf_msg->data, ipi_msg->data, len); + + /* received and processed interrupt ack */ + if (mbox_send_message(r5_core->rx_chan, NULL) < 0) + dev_err(r5_core->dev, "ack failed to mbox rx_chan\n"); + + schedule_work(&r5_core->mbox_work); +} + +/** + * zynqmp_r5_setup_mbox() - Setup mailboxes related properties + * this is used for each individual R5 core + * + * @r5_core: pointer to the ZynqMP r5 core data + * + * Function to setup mailboxes related properties + * + */ +static void zynqmp_r5_setup_mbox(struct zynqmp_r5_core *r5_core) +{ + struct zynqmp_r5_cluster *cluster; + struct mbox_client *mbox_cl; + + cluster = dev_get_drvdata(r5_core->dev->parent); + + /** + * ToDo: Use only one IPI channel for APU to communicate with both RPUs + * in split mode. As of now, two IPI channels are expeceted for APU + * to communicate with RPU. for example, APU(IPI0)<-> RPU0(IPI1) and + * APU(IPI7)<->RPU1(IPI2). However, this is not the optimized use + * of the hardware. As per hardware reference manual, each IPI channel + * can receive interrupt from another IPI channel. So APU must be able + * to communicate with both RPUs simultaneously using same IPI channel. + * For example, this is valid case: APU(IPI0)<->RPU0(IPI1) and + * APU(IPI0)<->RPU1(IPI2). However, with current available examples + * and RPU firmware, this configuration in device-tree is causing system-crash. + * And so, using extra IPI channel is required in device-tree. In split + * mode explicitly inform user about this limitation and requirement. + */ + if (cluster->mode == SPLIT_MODE) + dev_warn(r5_core->dev, "split mode: APU should use two IPI channels\n"); + + mbox_cl = &r5_core->mbox_cl; + mbox_cl->rx_callback = zynqmp_r5_mb_rx_cb; + mbox_cl->tx_block = false; + mbox_cl->knows_txdone = false; + mbox_cl->tx_done = NULL; + mbox_cl->dev = r5_core->dev; + + /* Request TX and RX channels */ + r5_core->tx_chan = mbox_request_channel_byname(mbox_cl, "tx"); + if (IS_ERR(r5_core->tx_chan)) { + r5_core->tx_chan = NULL; + return; + } + + r5_core->rx_chan = mbox_request_channel_byname(mbox_cl, "rx"); + if (IS_ERR(r5_core->rx_chan)) { + mbox_free_channel(r5_core->tx_chan); + r5_core->rx_chan = NULL; + r5_core->tx_chan = NULL; + return; + } + + INIT_WORK(&r5_core->mbox_work, handle_event_notified); +} + +static void zynqmp_r5_free_mbox(struct zynqmp_r5_core *r5_core) +{ + if (r5_core->tx_chan) { + mbox_free_channel(r5_core->tx_chan); + r5_core->tx_chan = NULL; + } + + if (r5_core->rx_chan) { + mbox_free_channel(r5_core->rx_chan); + r5_core->rx_chan = NULL; + } +} + +/* + * zynqmp_r5_core_kick() - kick a firmware if mbox is provided + * @rproc: r5 core's corresponding rproc structure + * @vqid: virtqueue ID + */ +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid) +{ + struct zynqmp_r5_core *r5_core = rproc->priv; + struct device *dev = r5_core->dev; + struct zynqmp_ipi_message *mb_msg; + int ret; + + /* don't handle kick if mbox setup failed for this core */ + if (!r5_core->tx_chan && !r5_core->rx_chan) + return; + + mb_msg = (struct zynqmp_ipi_message *)r5_core->tx_mc_buf; + memcpy(mb_msg->data, &vqid, sizeof(vqid)); + mb_msg->len = sizeof(vqid); + ret = mbox_send_message(r5_core->tx_chan, mb_msg); + if (ret < 0) + dev_warn(dev, "failed to send message\n"); +} + /* * zynqmp_r5_set_mode() * @@ -227,6 +423,63 @@ static int zynqmp_r5_mem_region_unmap(struct rproc *rproc, return 0; } +/** + * zynqmp_r5_get_mem_region_node() + * parse memory-region property and get reserved mem regions + * + * @r5_core: pointer to zynqmp_r5_core type object + * + * Return: 0 for success and error code for failure. + */ +static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) +{ + struct device_node *np, *rmem_np; + struct reserved_mem **rmem; + int res_mem_count, i; + struct device *dev; + + dev = r5_core->dev; + np = r5_core->np; + + res_mem_count = of_property_count_elems_of_size(np, "memory-region", + sizeof(phandle)); + + if (res_mem_count <= 0) { + dev_warn(dev, "failed to get memory-region property %d\n", + res_mem_count); + return 0; + } + + if (!r5_core->tx_chan && !r5_core->rx_chan) + res_mem_count = 1; + + rmem = devm_kcalloc(dev, res_mem_count, + sizeof(struct reserved_mem *), GFP_KERNEL); + if (!rmem) + return -ENOMEM; + + for (i = 0; i < res_mem_count; i++) { + rmem_np = of_parse_phandle(np, "memory-region", i); + if (!rmem_np) + goto release_rmem; + + rmem[i] = of_reserved_mem_lookup(rmem_np); + if (!rmem[i]) { + of_node_put(rmem_np); + goto release_rmem; + } + + of_node_put(rmem_np); + } + + r5_core->rmem_count = res_mem_count; + r5_core->rmem = rmem; + return 0; + +release_rmem: + return -EINVAL; +} + /* * add_mem_regions_carveout() * @rproc: single R5 core's corresponding rproc instance @@ -241,6 +494,7 @@ static int add_mem_regions_carveout(struct rproc *rproc) struct zynqmp_r5_core *r5_core; struct reserved_mem *rmem; int i, num_mem_regions; + const char *name; r5_core = (struct zynqmp_r5_core *)rproc->priv; num_mem_regions = r5_core->rmem_count; @@ -253,15 +507,33 @@ static int add_mem_regions_carveout(struct rproc *rproc) rproc_mem = rproc_of_resm_mem_entry_init(&rproc->dev, i, rmem->size, rmem->base, - rmem->name); + "vdev0buffer"); } else { + /* + * As per bindings 3rd entry in memory-region property + * must contain vring0 and 4th entry must contain vring1 + * memory-regions. For remoteproc framework it is + * required to have fixed names for these carveouts i.e. + * in the form of "vdev%dvring%d" where first %d is ID + * of vdev and second %d is ID of vring. Assign fix names + * instead of node names, as node names may contain + * @unit-address as well i.e. vdev0vring0@xxxxxxxx which + * won't work. + */ + if (!strncmp(rmem->name, "vdev0vring0", strlen("vdev0vring0"))) + name = "vdev0vring0"; + else if (!strncmp(rmem->name, "vdev0vring1", strlen("vdev0vring1"))) + name = "vdev0vring1"; + else + name = rmem->name; + /* Register associated reserved memory regions */ rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL, (dma_addr_t)rmem->base, rmem->size, rmem->base, zynqmp_r5_mem_region_map, zynqmp_r5_mem_region_unmap, - rmem->name); + name); } if (!rproc_mem) @@ -572,6 +844,20 @@ static int zynqmp_r5_rproc_prepare(struct rproc *rproc) return ret; } + /* + * If mailbox nodes are disabled using "status" property then setting up + * mailbox channels will be failed and in that case, we don't need vrings + * and vdevbuffer for this core. So, setup mailbox before parsing + * memory-region property. If "tx" and "rx" mailboxes are not setup, then + * only parse and add first memory-region carveout. As per bindings, it + * must be firmware load region + */ + zynqmp_r5_setup_mbox(rproc->priv); + + ret = zynqmp_r5_get_mem_region_node(rproc->priv); + if (ret) + dev_warn(&rproc->dev, "memory-region prop failed %d\n", ret); + ret = add_mem_regions_carveout(rproc); if (ret) { dev_err(&rproc->dev, "failed to get reserve mem regions %d\n", ret); @@ -597,6 +883,8 @@ static int zynqmp_r5_rproc_unprepare(struct rproc *rproc) r5_core = (struct zynqmp_r5_core *)rproc->priv; + zynqmp_r5_free_mbox(r5_core); + for (i = 0; i < r5_core->tcm_bank_count; i++) { pm_domain_id = r5_core->tcm_banks[i]->pm_domain_id; if (zynqmp_pm_release_node(pm_domain_id)) @@ -617,6 +905,7 @@ static const struct rproc_ops zynqmp_r5_rproc_ops = { .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table, .sanity_check = rproc_elf_sanity_check, .get_boot_addr = rproc_elf_get_boot_addr, + .kick = zynqmp_r5_rproc_kick, }; /** @@ -726,59 +1015,6 @@ static int zynqmp_r5_get_tcm_node(struct zynqmp_r5_cluster *cluster) return 0; } -/** - * zynqmp_r5_get_mem_region_node() - * parse memory-region property and get reserved mem regions - * - * @r5_core: pointer to zynqmp_r5_core type object - * - * Return: 0 for success and error code for failure. - */ -static int zynqmp_r5_get_mem_region_node(struct zynqmp_r5_core *r5_core) -{ - struct device_node *np, *rmem_np; - struct reserved_mem **rmem; - int res_mem_count, i; - struct device *dev; - - dev = r5_core->dev; - np = r5_core->np; - - res_mem_count = of_property_count_elems_of_size(np, "memory-region", - sizeof(phandle)); - if (res_mem_count <= 0) { - dev_warn(dev, "failed to get memory-region property %d\n", - res_mem_count); - return 0; - } - - rmem = devm_kcalloc(dev, res_mem_count, - sizeof(struct reserved_mem *), GFP_KERNEL); - if (!rmem) - return -ENOMEM; - - for (i = 0; i < res_mem_count; i++) { - rmem_np = of_parse_phandle(np, "memory-region", i); - if (!rmem_np) - goto release_rmem; - - rmem[i] = of_reserved_mem_lookup(rmem_np); - if (!rmem[i]) { - of_node_put(rmem_np); - goto release_rmem; - } - - of_node_put(rmem_np); - } - - r5_core->rmem_count = res_mem_count; - r5_core->rmem = rmem; - return 0; - -release_rmem: - return -EINVAL; -} - /* * zynqmp_r5_core_init() * Create and initialize zynqmp_r5_core type object @@ -806,10 +1042,6 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, for (i = 0; i < cluster->core_count; i++) { r5_core = cluster->r5_cores[i]; - ret = zynqmp_r5_get_mem_region_node(r5_core); - if (ret) - dev_warn(dev, "memory-region prop failed %d\n", ret); - /* Initialize r5 cores with power-domains parsed from dts */ ret = of_property_read_u32_index(r5_core->np, "power-domains", 1, &r5_core->pm_domain_id);
This patch makes each r5 core mailbox client and uses tx and rx channels to send and receive data to/from remote processor respectively. This is needed for rpmsg communication to remote processor. Signed-off-by: Tanmay Shah <tanmay.shah@amd.com> --- Changes in v2: - fix vrings carveout names as expeceted by remoteproc framework drivers/remoteproc/xlnx_r5_remoteproc.c | 352 ++++++++++++++++++++---- 1 file changed, 292 insertions(+), 60 deletions(-) base-commit: 10de8156ed71d3dbd7e9099aa76e67ea2c37d4ff