@@ -7,4 +7,5 @@ obj-$(CONFIG_QAIC) := qaic.o
qaic-y := \
qaic_drv.o \
- mhi_controller.o
+ mhi_controller.o \
+ qaic_control.o
@@ -7,11 +7,21 @@
#define QAICINTERNAL_H_
#include <linux/cdev.h>
+#include <linux/idr.h>
#include <linux/kref.h>
#include <linux/mhi.h>
#include <linux/mutex.h>
#include <linux/pci.h>
+#include <linux/spinlock.h>
#include <linux/srcu.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+
+#define QAIC_NUM_DBC 16
+#define QAIC_DBC_BASE 0x20000
+#define QAIC_DBC_SIZE 0x1000
+
+#define QAIC_DBC_OFF(i) ((i) * QAIC_DBC_SIZE + QAIC_DBC_BASE)
struct qaic_user {
pid_t handle;
@@ -21,17 +31,68 @@ struct qaic_user {
struct kref ref_count;
};
+struct dma_bridge_chan {
+ struct qaic_device *qdev;
+ unsigned int id;
+ /* also the base of the entire memory allocation */
+ void *req_q_base;
+ void *rsp_q_base;
+ dma_addr_t dma_addr;
+ u32 total_size;
+ u32 nelem;
+ struct mutex mem_lock;
+ struct idr mem_handles;
+ struct qaic_user *usr;
+ u16 next_req_id;
+ void __iomem *dbc_base;
+ spinlock_t xfer_lock;
+ struct list_head xfer_list;
+ struct srcu_struct ch_lock;
+ struct dentry *debugfs_root;
+ bool in_use;
+ wait_queue_head_t dbc_release;
+};
+
struct qaic_device {
struct pci_dev *pdev;
int bars;
void __iomem *bar_0;
+ void __iomem *bar_2;
struct mhi_controller *mhi_cntl;
struct mhi_device *cntl_ch;
+ struct list_head cntl_xfer_list;
+ u32 next_seq_num;
+ struct mutex cntl_mutex;
+ bool cntl_lost_buf;
struct cdev *cdev;
struct device *dev;
+ struct dma_bridge_chan dbc[QAIC_NUM_DBC];
+ struct workqueue_struct *cntl_wq;
bool in_reset;
struct srcu_struct dev_lock;
struct list_head users;
struct mutex users_mutex;
};
+
+int get_dbc_req_elem_size(void);
+int get_dbc_rsp_elem_size(void);
+int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr,
+ u16 *major, u16 *minor);
+int qaic_manage_ioctl(struct qaic_device *qdev, struct qaic_user *usr,
+ unsigned long arg);
+
+void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result);
+
+void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result);
+
+int qaic_control_open(struct qaic_device *qdev);
+void qaic_control_close(struct qaic_device *qdev);
+void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr);
+
+int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr);
+void release_dbc(struct qaic_device *qdev, u32 dbc_id);
+
+void wake_all_cntl(struct qaic_device *qdev);
#endif /* QAICINTERNAL_H_ */
new file mode 100644
@@ -0,0 +1,1015 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. */
+
+#include <asm/byteorder.h>
+#include <linux/completion.h>
+#include <linux/dma-mapping.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/mhi.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/workqueue.h>
+#include <linux/wait.h>
+#include <uapi/misc/qaic.h>
+
+#include "qaic.h"
+
+#define MANAGE_MAGIC_NUMBER 0x43494151 /* "QAIC" in little endian */
+#define QAIC_DBC_Q_GAP 0x100
+#define QAIC_DBC_Q_BUF_ALIGN 0x1000
+#define RESP_TIMEOUT (60 * HZ)
+
+/*
+ * Wire encoding structures for the manage protocol.
+ * All fields are little endian on the wire
+ */
+struct _msg_hdr {
+ u32 magic_number;
+ u32 sequence_number;
+ u32 len; /* length of this message */
+ u32 count; /* number of transactions in this message */
+ u32 handle; /* unique id to track the resources consumed */
+} __packed;
+
+struct _msg {
+ struct _msg_hdr hdr;
+ u8 data[QAIC_MANAGE_MAX_MSG_LENGTH];
+} __packed;
+
+struct wrapper_msg {
+ struct kref ref_count;
+ struct _msg msg;
+};
+
+struct _trans_hdr {
+ u32 type;
+ u32 len;
+} __packed;
+
+struct _trans_passthrough {
+ struct _trans_hdr hdr;
+ u8 data[0];
+} __packed;
+
+struct _addr_size_pair {
+ u64 addr;
+ u64 size;
+} __packed;
+
+struct _trans_dma_xfer {
+ struct _trans_hdr hdr;
+ u32 tag;
+ u32 count;
+ struct _addr_size_pair data[0];
+} __packed;
+
+struct _trans_activate_to_dev {
+ struct _trans_hdr hdr;
+ u32 buf_len;
+ u64 req_q_addr;
+ u32 req_q_size;
+ u64 rsp_q_addr;
+ u32 rsp_q_size;
+ u32 reserved;
+} __packed;
+
+struct _trans_activate_from_dev {
+ struct _trans_hdr hdr;
+ u32 status;
+ u32 dbc_id;
+} __packed;
+
+struct _trans_deactivate_from_dev {
+ struct _trans_hdr hdr;
+ u32 status;
+ u32 dbc_id;
+} __packed;
+
+struct _trans_terminate_to_dev {
+ struct _trans_hdr hdr;
+ u32 handle;
+} __packed;
+
+struct _trans_terminate_from_dev {
+ struct _trans_hdr hdr;
+ u32 status;
+} __packed;
+
+struct _trans_status_to_dev {
+ struct _trans_hdr hdr;
+} __packed;
+
+struct _trans_status_from_dev {
+ struct _trans_hdr hdr;
+ u16 major;
+ u16 minor;
+ u32 status;
+} __packed;
+
+struct xfer_queue_elem {
+ struct list_head list;
+ u32 seq_num;
+ struct completion xfer_done;
+ void *buf;
+};
+
+struct dma_xfer {
+ struct list_head list;
+ struct sg_table *sgt;
+ struct page **page_list;
+ unsigned long nr_pages;
+};
+
+struct ioctl_resources {
+ struct list_head dma_xfers;
+ void *buf;
+ dma_addr_t dma_addr;
+ u32 total_size;
+ u32 nelem;
+ void *rsp_q_base;
+ u32 status;
+ u32 dbc_id;
+};
+
+struct resp_work {
+ struct work_struct work;
+ struct qaic_device *qdev;
+ void *buf;
+};
+
+static void free_wrapper(struct kref *ref)
+{
+ struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg,
+ ref_count);
+
+ kfree(wrapper);
+}
+
+static void save_dbc_buf(struct qaic_device *qdev,
+ struct ioctl_resources *resources,
+ struct qaic_user *usr)
+{
+ u32 dbc_id = resources->dbc_id;
+
+ if (resources->buf) {
+ wait_event_interruptible(qdev->dbc[dbc_id].dbc_release,
+ !qdev->dbc[dbc_id].in_use);
+ qdev->dbc[dbc_id].req_q_base = resources->buf;
+ qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;
+ qdev->dbc[dbc_id].dma_addr = resources->dma_addr;
+ qdev->dbc[dbc_id].total_size = resources->total_size;
+ qdev->dbc[dbc_id].nelem = resources->nelem;
+ qdev->dbc[dbc_id].usr = usr;
+ qdev->dbc[dbc_id].in_use = true;
+ resources->buf = 0;
+ }
+}
+
+static void free_dbc_buf(struct qaic_device *qdev,
+ struct ioctl_resources *resources)
+{
+ if (resources->buf)
+ dma_free_coherent(&qdev->pdev->dev, resources->total_size,
+ resources->buf, resources->dma_addr);
+ resources->buf = 0;
+}
+
+static void free_dma_xfers(struct qaic_device *qdev,
+ struct ioctl_resources *resources)
+{
+ struct dma_xfer *xfer;
+ struct dma_xfer *x;
+ int i;
+
+ list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {
+ dma_unmap_sg(&qdev->pdev->dev, xfer->sgt->sgl, xfer->sgt->nents,
+ DMA_TO_DEVICE);
+ sg_free_table(xfer->sgt);
+ kfree(xfer->sgt);
+ for (i = 0; i < xfer->nr_pages; ++i)
+ put_page(xfer->page_list[i]);
+ kfree(xfer->page_list);
+ list_del(&xfer->list);
+ kfree(xfer);
+ }
+}
+
+static int encode_passthrough(struct qaic_device *qdev, void *trans,
+ struct _msg *msg, u32 *user_len)
+{
+ struct qaic_manage_trans_passthrough *in_trans = trans;
+ struct _trans_passthrough *out_trans = (void *)msg + msg->hdr.len;
+
+ if (msg->hdr.len + in_trans->hdr.len > sizeof(*msg))
+ return -ENOSPC;
+
+ memcpy(out_trans, in_trans, in_trans->hdr.len);
+ msg->hdr.len += in_trans->hdr.len;
+ *user_len += in_trans->hdr.len;
+ out_trans->hdr.type = cpu_to_le32(TRANS_PASSTHROUGH_TO_DEV);
+ out_trans->hdr.len = cpu_to_le32(out_trans->hdr.len);
+
+ return 0;
+}
+
+static int encode_dma(struct qaic_device *qdev, void *trans, struct _msg *msg,
+ u32 *user_len, struct ioctl_resources *resources)
+{
+ struct qaic_manage_trans_dma_xfer *in_trans = trans;
+ struct _trans_dma_xfer *out_trans = (void *)msg + msg->hdr.len;
+ struct dma_xfer *xfer;
+ unsigned long nr_pages;
+ struct page **page_list;
+ struct scatterlist *last;
+ struct scatterlist *sg;
+ struct sg_table *sgt;
+ unsigned int dma_len;
+ int nents;
+ int dmas;
+ int ret;
+ int i;
+
+ if (in_trans->addr + in_trans->size < in_trans->addr ||
+ !in_trans->size) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);
+ if (!xfer) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ nr_pages = PAGE_ALIGN(in_trans->size + offset_in_page(in_trans->addr))
+ >> PAGE_SHIFT;
+
+ page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL);
+ if (!page_list) {
+ ret = -ENOMEM;
+ goto free_resource;
+ }
+
+ ret = get_user_pages_fast(in_trans->addr, nr_pages, 0, page_list);
+ if (ret < 0 || ret != nr_pages) {
+ ret = -EFAULT;
+ goto free_page_list;
+ }
+
+ sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
+ if (!sgt) {
+ ret = -ENOMEM;
+ goto put_pages;
+ }
+
+ ret = sg_alloc_table_from_pages(sgt, page_list, nr_pages,
+ offset_in_page(in_trans->addr),
+ in_trans->size, GFP_KERNEL);
+ if (ret) {
+ ret = -ENOMEM;
+ goto free_sgt;
+ }
+
+ nents = dma_map_sg(&qdev->pdev->dev, sgt->sgl, sgt->nents,
+ DMA_TO_DEVICE);
+ if (!nents) {
+ ret = -EFAULT;
+ goto free_table;
+ }
+
+ /*
+ * It turns out several of the iommu drivers don't combine adjacent
+ * regions, which is really what we expect based on the description of
+ * dma_map_sg(), so lets see if that can be done. It makes our message
+ * more efficent.
+ */
+ dmas = 0;
+ last = sgt->sgl;
+ for_each_sg(sgt->sgl, sg, nents, i) {
+ if (sg_dma_address(last) + sg_dma_len(last) !=
+ sg_dma_address(sg))
+ dmas++;
+ last = sg;
+ }
+
+ /*
+ * now that we finally know how many memory segments we will be encoding
+ * we can check to see if we have space in the message
+ */
+ if (msg->hdr.len + sizeof(*out_trans) + dmas * sizeof(*out_trans->data)
+ > sizeof(*msg)) {
+ ret = -ENOSPC;
+ goto dma_unmap;
+ }
+
+ msg->hdr.len += sizeof(*out_trans) + dmas * sizeof(*out_trans->data);
+
+ out_trans->hdr.type = cpu_to_le32(TRANS_DMA_XFER_TO_DEV);
+ out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans) +
+ dmas * sizeof(*out_trans->data));
+ out_trans->tag = cpu_to_le32(in_trans->tag);
+ out_trans->count = cpu_to_le32(dmas);
+
+ i = 0;
+ last = sgt->sgl;
+ dma_len = 0;
+ for_each_sg(sgt->sgl, sg, nents, dmas) {
+ /* hit a discontinuity, finalize segment and start new one */
+ if (sg_dma_address(last) + sg_dma_len(last) !=
+ sg_dma_address(sg)) {
+ out_trans->data[i].size = cpu_to_le64(dma_len);
+ if (dma_len)
+ i++;
+ dma_len = 0;
+ out_trans->data[i].addr =
+ cpu_to_le64(sg_dma_address(sg));
+ }
+ dma_len += sg_dma_len(sg);
+ last = sg;
+ }
+ /* finalize the last segment */
+ out_trans->data[i].size = cpu_to_le64(dma_len);
+
+ *user_len += in_trans->hdr.len;
+
+ xfer->sgt = sgt;
+ xfer->page_list = page_list;
+ xfer->nr_pages = nr_pages;
+ list_add(&xfer->list, &resources->dma_xfers);
+ return 0;
+
+dma_unmap:
+ dma_unmap_sg(&qdev->pdev->dev, sgt->sgl, sgt->nents, DMA_TO_DEVICE);
+free_table:
+ sg_free_table(sgt);
+free_sgt:
+ kfree(sgt);
+put_pages:
+ for (i = 0; i < nr_pages; ++i)
+ put_page(page_list[i]);
+free_page_list:
+ kfree(page_list);
+free_resource:
+ kfree(xfer);
+out:
+ return ret;
+}
+
+static int encode_activate(struct qaic_device *qdev, void *trans,
+ struct _msg *msg, u32 *user_len,
+ struct ioctl_resources *resources)
+{
+ struct qaic_manage_trans_activate_to_dev *in_trans = trans;
+ struct _trans_activate_to_dev *out_trans = (void *)msg + msg->hdr.len;
+ dma_addr_t dma_addr;
+ void *buf;
+ u32 nelem;
+ u32 size;
+
+ if (msg->hdr.len + sizeof(*out_trans) > sizeof(*msg))
+ return -ENOSPC;
+
+ if (!in_trans->queue_size)
+ return -EINVAL;
+
+ if (in_trans->resv)
+ return -EINVAL;
+
+ nelem = in_trans->queue_size;
+ size = (get_dbc_req_elem_size() + get_dbc_rsp_elem_size()) * nelem;
+ if (size / nelem != get_dbc_req_elem_size() + get_dbc_rsp_elem_size())
+ return -EINVAL;
+
+ if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)
+ return -EINVAL;
+
+ size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);
+
+ buf = dma_alloc_coherent(&qdev->pdev->dev, size, &dma_addr, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ out_trans->hdr.type = cpu_to_le32(TRANS_ACTIVATE_TO_DEV);
+ out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));
+ out_trans->buf_len = cpu_to_le32(size);
+ out_trans->req_q_addr = cpu_to_le64(dma_addr);
+ out_trans->req_q_size = cpu_to_le32(nelem);
+ out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem *
+ get_dbc_rsp_elem_size());
+ out_trans->rsp_q_size = cpu_to_le32(nelem);
+
+ *user_len += in_trans->hdr.len;
+ msg->hdr.len += sizeof(*out_trans);
+
+ resources->buf = buf;
+ resources->dma_addr = dma_addr;
+ resources->total_size = size;
+ resources->nelem = nelem;
+ resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();
+ return 0;
+}
+
+static int encode_deactivate(struct qaic_device *qdev, void *trans,
+ u32 *user_len, struct qaic_user *usr)
+{
+ struct qaic_manage_trans_deactivate *in_trans = trans;
+
+ if (in_trans->dbc_id >= QAIC_NUM_DBC || in_trans->resv)
+ return -EINVAL;
+
+ *user_len += in_trans->hdr.len;
+
+ return disable_dbc(qdev, in_trans->dbc_id, usr);
+}
+
+static int encode_status(struct qaic_device *qdev, void *trans,
+ struct _msg *msg, u32 *user_len)
+{
+ struct qaic_manage_trans_status_to_dev *in_trans = trans;
+ struct _trans_status_to_dev *out_trans = (void *)msg + msg->hdr.len;
+
+ if (msg->hdr.len + in_trans->hdr.len > sizeof(*msg))
+ return -ENOSPC;
+
+ out_trans->hdr.type = cpu_to_le32(TRANS_STATUS_TO_DEV);
+ out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
+ msg->hdr.len += in_trans->hdr.len;
+ *user_len += in_trans->hdr.len;
+
+ return 0;
+}
+static int encode_message(struct qaic_device *qdev,
+ struct qaic_manage_msg *user_msg, struct _msg *msg,
+ struct ioctl_resources *resources,
+ struct qaic_user *usr)
+{
+ struct qaic_manage_trans_hdr *trans_hdr;
+ u32 user_len = 0;
+ int ret;
+ int i;
+
+ msg->hdr.len = sizeof(msg->hdr);
+ for (i = 0; i < user_msg->count; ++i) {
+ if (user_len >= user_msg->len) {
+ ret = -EINVAL;
+ break;
+ }
+ trans_hdr = (struct qaic_manage_trans_hdr *)
+ (user_msg->data + user_len);
+ if (user_len + trans_hdr->len > user_msg->len) {
+ ret = -EINVAL;
+ break;
+ }
+
+ switch (trans_hdr->type) {
+ case TRANS_PASSTHROUGH_FROM_USR:
+ ret = encode_passthrough(qdev, trans_hdr, msg,
+ &user_len);
+ break;
+ case TRANS_DMA_XFER_FROM_USR:
+ ret = encode_dma(qdev, trans_hdr, msg, &user_len,
+ resources);
+ break;
+ case TRANS_ACTIVATE_FROM_USR:
+ ret = encode_activate(qdev, trans_hdr, msg, &user_len,
+ resources);
+ break;
+ case TRANS_DEACTIVATE_FROM_USR:
+ ret = encode_deactivate(qdev, trans_hdr, &user_len,
+ usr);
+ break;
+ case TRANS_STATUS_FROM_USR:
+ ret = encode_status(qdev, trans_hdr, msg, &user_len);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ if (ret)
+ break;
+ }
+
+ if (user_len != user_msg->len)
+ ret = -EINVAL;
+
+ if (ret) {
+ free_dma_xfers(qdev, resources);
+ free_dbc_buf(qdev, resources);
+ return ret;
+ }
+
+ msg->hdr.count = user_msg->count;
+ return 0;
+}
+
+static int decode_passthrough(struct qaic_device *qdev, void *trans,
+ struct qaic_manage_msg *user_msg, u32 *msg_len)
+{
+ struct _trans_passthrough *in_trans = trans;
+ struct qaic_manage_trans_passthrough *out_trans;
+ u32 len;
+
+ out_trans = (void *)user_msg->data + user_msg->len;
+
+ len = le32_to_cpu(in_trans->hdr.len);
+ if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
+ return -ENOSPC;
+
+ memcpy(out_trans, in_trans, len);
+ user_msg->len += len;
+ *msg_len += len;
+ out_trans->hdr.type = le32_to_cpu(out_trans->hdr.type);
+ return 0;
+}
+
+static int decode_activate(struct qaic_device *qdev, void *trans,
+ struct qaic_manage_msg *user_msg, u32 *msg_len,
+ struct ioctl_resources *resources,
+ struct qaic_user *usr)
+{
+ struct _trans_activate_from_dev *in_trans = trans;
+ struct qaic_manage_trans_activate_from_dev *out_trans;
+ u32 len;
+
+ out_trans = (void *)user_msg->data + user_msg->len;
+
+ len = le32_to_cpu(in_trans->hdr.len);
+ if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
+ return -ENOSPC;
+
+ user_msg->len += len;
+ *msg_len += len;
+ out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
+ out_trans->hdr.len = len;
+ out_trans->status = le32_to_cpu(in_trans->status);
+ out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);
+
+ if (!resources->buf)
+ /* how did we get an activate response without a request? */
+ return -EINVAL;
+
+ if (out_trans->dbc_id >= QAIC_NUM_DBC)
+ /*
+ * The device assigned an invalid resource, which should never
+ * happen. Inject an error so the user can try to recover.
+ */
+ out_trans->status = -ENODEV;
+
+ resources->status = out_trans->status;
+ resources->dbc_id = out_trans->dbc_id;
+ if (!resources->status)
+ save_dbc_buf(qdev, resources, usr);
+ return 0;
+}
+
+static int decode_deactivate(struct qaic_device *qdev, void *trans,
+ u32 *msg_len)
+{
+ struct _trans_deactivate_from_dev *in_trans = trans;
+ u32 dbc_id = le32_to_cpu(in_trans->dbc_id);
+ u32 status = le32_to_cpu(in_trans->status);
+
+ if (dbc_id >= QAIC_NUM_DBC)
+ /*
+ * The device assigned an invalid resource, which should never
+ * happen. Inject an error so the user can try to recover.
+ */
+ return -ENODEV;
+
+ if (status)
+ /*
+ * Releasing resources failed on the device side, which puts
+ * us in a bind since they may still be in use, so be safe and
+ * do nothing.
+ */
+ return -ENODEV;
+
+ release_dbc(qdev, dbc_id);
+ *msg_len += sizeof(*in_trans);
+ return 0;
+}
+
+static int decode_status(struct qaic_device *qdev, void *trans,
+ struct qaic_manage_msg *user_msg, u32 *user_len)
+{
+ struct _trans_status_from_dev *in_trans = trans;
+ struct qaic_manage_trans_status_from_dev *out_trans;
+ u32 len;
+
+ out_trans = (void *)user_msg->data + user_msg->len;
+
+ len = le32_to_cpu(in_trans->hdr.len);
+ if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
+ return -ENOSPC;
+
+ out_trans->hdr.type = le32_to_cpu(TRANS_STATUS_FROM_DEV);
+ out_trans->hdr.len = len;
+ out_trans->major = le32_to_cpu(in_trans->major);
+ out_trans->minor = le32_to_cpu(in_trans->minor);
+ *user_len += in_trans->hdr.len;
+ user_msg->len += len;
+
+ return 0;
+}
+
+static int decode_message(struct qaic_device *qdev,
+ struct qaic_manage_msg *user_msg, struct _msg *msg,
+ struct ioctl_resources *resources,
+ struct qaic_user *usr)
+{
+ struct _trans_hdr *trans_hdr;
+ u32 msg_len = 0;
+ int ret;
+ int i;
+
+ if (msg->hdr.len > sizeof(*msg))
+ return -EINVAL;
+
+ user_msg->len = 0;
+ user_msg->count = le32_to_cpu(msg->hdr.count);
+
+ for (i = 0; i < user_msg->count; ++i) {
+ trans_hdr = (struct _trans_hdr *)(msg->data + msg_len);
+ if (msg_len + trans_hdr->len > msg->hdr.len)
+ return -EINVAL;
+
+ switch (trans_hdr->type) {
+ case TRANS_PASSTHROUGH_FROM_DEV:
+ ret = decode_passthrough(qdev, trans_hdr, user_msg,
+ &msg_len);
+ break;
+ case TRANS_ACTIVATE_FROM_DEV:
+ ret = decode_activate(qdev, trans_hdr, user_msg,
+ &msg_len, resources, usr);
+ break;
+ case TRANS_DEACTIVATE_FROM_DEV:
+ ret = decode_deactivate(qdev, trans_hdr, &msg_len);
+ break;
+ case TRANS_STATUS_FROM_DEV:
+ ret = decode_status(qdev, trans_hdr, user_msg,
+ &msg_len);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (ret)
+ return ret;
+ }
+
+ if (msg_len != (msg->hdr.len - sizeof(msg->hdr)))
+ return -EINVAL;
+
+ return 0;
+}
+
+static void *msg_xfer(struct qaic_device *qdev, struct wrapper_msg *wrapper,
+ u32 seq_num, bool ignore_signal)
+{
+ struct xfer_queue_elem elem;
+ struct _msg *out_buf;
+ size_t in_len;
+ long ret;
+
+ if (qdev->in_reset) {
+ mutex_unlock(&qdev->cntl_mutex);
+ return ERR_PTR(-ENODEV);
+ }
+
+ in_len = sizeof(wrapper->msg);
+
+ elem.seq_num = seq_num;
+ elem.buf = NULL;
+ init_completion(&elem.xfer_done);
+ if (likely(!qdev->cntl_lost_buf)) {
+ out_buf = kmalloc(sizeof(*out_buf), GFP_KERNEL);
+ if (!out_buf) {
+ mutex_unlock(&qdev->cntl_mutex);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ ret = mhi_queue_buf(qdev->cntl_ch, DMA_FROM_DEVICE,
+ out_buf, sizeof(*out_buf), MHI_EOT);
+ if (ret) {
+ mutex_unlock(&qdev->cntl_mutex);
+ return ERR_PTR(ret);
+ }
+ } else {
+ /*
+ * we lost a buffer because we queued a recv buf, but then
+ * queuing the corresponding tx buf failed. To try to avoid
+ * a memory leak, lets reclaim it and use it for this
+ * transaction.
+ */
+ qdev->cntl_lost_buf = false;
+ }
+
+ kref_get(&wrapper->ref_count);
+ ret = mhi_queue_buf(qdev->cntl_ch, DMA_TO_DEVICE, &wrapper->msg, in_len,
+ MHI_EOT);
+ if (ret) {
+ qdev->cntl_lost_buf = true;
+ kref_put(&wrapper->ref_count, free_wrapper);
+ mutex_unlock(&qdev->cntl_mutex);
+ return ERR_PTR(ret);
+ }
+
+ list_add_tail(&elem.list, &qdev->cntl_xfer_list);
+ mutex_unlock(&qdev->cntl_mutex);
+
+ if (ignore_signal)
+ ret = wait_for_completion_timeout(&elem.xfer_done,
+ RESP_TIMEOUT);
+ else
+ ret = wait_for_completion_interruptible_timeout(&elem.xfer_done,
+ RESP_TIMEOUT);
+ /*
+ * not using _interruptable because we have to cleanup or we'll
+ * likely cause memory corruption
+ */
+ mutex_lock(&qdev->cntl_mutex);
+ if (!list_empty(&elem.list))
+ list_del(&elem.list);
+ if (!ret && !elem.buf)
+ ret = -ETIMEDOUT;
+ else if (ret > 0 && !elem.buf)
+ ret = -EIO;
+ mutex_unlock(&qdev->cntl_mutex);
+
+ if (ret < 0) {
+ kfree(elem.buf);
+ return ERR_PTR(ret);
+ }
+
+ return elem.buf;
+}
+
+static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr,
+ struct qaic_manage_msg *user_msg)
+{
+ struct ioctl_resources resources;
+ struct wrapper_msg *wrapper;
+ struct _msg *msg;
+ struct _msg *rsp;
+ int ret;
+
+ INIT_LIST_HEAD(&resources.dma_xfers);
+ resources.buf = NULL;
+
+ if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||
+ user_msg->count >
+ QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr)) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ wrapper = kzalloc(sizeof(*wrapper), GFP_KERNEL);
+ if (!wrapper) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ kref_init(&wrapper->ref_count);
+ msg = &wrapper->msg;
+
+ ret = encode_message(qdev, user_msg, msg, &resources, usr);
+ if (ret)
+ goto encode_failed;
+
+ ret = mutex_lock_interruptible(&qdev->cntl_mutex);
+ if (ret)
+ goto lock_failed;
+ msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
+ msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
+ msg->hdr.len = cpu_to_le32(msg->hdr.len);
+ msg->hdr.count = cpu_to_le32(msg->hdr.count);
+ if (usr)
+ msg->hdr.handle = cpu_to_le32(usr->handle);
+ else
+ msg->hdr.handle = 0;
+
+ /* msg_xfer releases the mutex */
+ rsp = msg_xfer(qdev, wrapper, qdev->next_seq_num - 1, false);
+ if (IS_ERR(rsp)) {
+ ret = PTR_ERR(rsp);
+ goto lock_failed;
+ }
+
+ ret = decode_message(qdev, user_msg, rsp, &resources, usr);
+
+ kfree(rsp);
+lock_failed:
+ free_dma_xfers(qdev, &resources);
+ free_dbc_buf(qdev, &resources);
+encode_failed:
+ kref_put(&wrapper->ref_count, free_wrapper);
+out:
+ return ret;
+}
+
+int qaic_manage_ioctl(struct qaic_device *qdev, struct qaic_user *usr,
+ unsigned long arg)
+{
+ struct qaic_manage_msg *user_msg;
+ int ret;
+
+ user_msg = kmalloc(sizeof(*user_msg), GFP_KERNEL);
+ if (!user_msg) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ if (copy_from_user(user_msg, (void __user *)arg, sizeof(*user_msg))) {
+ ret = -EFAULT;
+ goto copy_from_user_failed;
+ }
+
+ ret = qaic_manage(qdev, usr, user_msg);
+ if (ret)
+ goto copy_from_user_failed;
+
+ if (copy_to_user((void __user *)arg, user_msg, sizeof(*user_msg)))
+ ret = -EFAULT;
+
+copy_from_user_failed:
+ kfree(user_msg);
+out:
+ return ret;
+}
+
+int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr,
+ u16 *major, u16 *minor)
+{
+ int ret;
+ struct qaic_manage_msg *user_msg;
+ struct qaic_manage_trans_status_to_dev *status_query;
+ struct qaic_manage_trans_status_from_dev *status_result;
+
+ user_msg = kmalloc(sizeof(*user_msg), GFP_KERNEL);
+ if (!user_msg) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ user_msg->len = sizeof(*status_query);
+ user_msg->count = 1;
+
+ status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;
+ status_query->hdr.type = TRANS_STATUS_FROM_USR;
+ status_query->hdr.len = sizeof(status_query->hdr);
+
+ ret = qaic_manage(qdev, usr, user_msg);
+ if (ret)
+ goto kfree_user_msg;
+ status_result =
+ (struct qaic_manage_trans_status_from_dev *)user_msg->data;
+ *major = status_result->major;
+ *minor = status_result->minor;
+
+kfree_user_msg:
+ kfree(user_msg);
+out:
+ return ret;
+}
+
+static void resp_worker(struct work_struct *work)
+{
+ struct resp_work *resp = container_of(work, struct resp_work, work);
+ struct qaic_device *qdev = resp->qdev;
+ struct _msg *msg = resp->buf;
+ struct xfer_queue_elem *elem;
+ struct xfer_queue_elem *i;
+ bool found = false;
+
+ if (msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {
+ kfree(msg);
+ kfree(resp);
+ return;
+ }
+
+ mutex_lock(&qdev->cntl_mutex);
+ list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
+ if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {
+ found = true;
+ list_del_init(&elem->list);
+ elem->buf = msg;
+ complete_all(&elem->xfer_done);
+ break;
+ }
+ }
+ mutex_unlock(&qdev->cntl_mutex);
+
+ if (!found)
+ /* request must have timed out, drop packet */
+ kfree(msg);
+
+ kfree(resp);
+}
+
+void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct _msg *msg = mhi_result->buf_addr;
+ struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg,
+ msg);
+
+ kref_put(&wrapper->ref_count, free_wrapper);
+}
+
+void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);
+ struct _msg *msg = mhi_result->buf_addr;
+ struct resp_work *resp;
+
+ if (mhi_result->transaction_status) {
+ kfree(msg);
+ return;
+ }
+
+ resp = kmalloc(sizeof(*resp), GFP_ATOMIC);
+ if (!resp) {
+ pci_err(qdev->pdev, "dl_xfer_cb alloc fail, dropping message\n");
+ kfree(msg);
+ return;
+ }
+
+ INIT_WORK(&resp->work, resp_worker);
+ resp->qdev = qdev;
+ resp->buf = msg;
+ queue_work(qdev->cntl_wq, &resp->work);
+}
+
+int qaic_control_open(struct qaic_device *qdev)
+{
+ if (!qdev->cntl_ch)
+ return -ENODEV;
+
+ return mhi_prepare_for_transfer(qdev->cntl_ch);
+}
+
+void qaic_control_close(struct qaic_device *qdev)
+{
+ mhi_unprepare_from_transfer(qdev->cntl_ch);
+}
+
+void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)
+{
+ struct _trans_terminate_to_dev *trans;
+ struct wrapper_msg *wrapper;
+ struct _msg *msg;
+ struct _msg *rsp;
+
+ wrapper = kzalloc(sizeof(*wrapper), GFP_KERNEL);
+ if (!wrapper)
+ return;
+
+ kref_init(&wrapper->ref_count);
+ msg = &wrapper->msg;
+
+ trans = (struct _trans_terminate_to_dev *)msg->data;
+
+ trans->hdr.type = cpu_to_le32(TRANS_TERMINATE_TO_DEV);
+ trans->hdr.len = cpu_to_le32(sizeof(*trans));
+ trans->handle = cpu_to_le32(usr->handle);
+
+ mutex_lock(&qdev->cntl_mutex);
+ msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
+ msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
+ msg->hdr.len = cpu_to_le32(sizeof(msg->hdr) + sizeof(*trans));
+ msg->hdr.count = cpu_to_le32(1);
+ msg->hdr.handle = cpu_to_le32(usr->handle);
+
+ /*
+ * msg_xfer releases the mutex
+ * We don't care about the return of msg_xfer since we will not do
+ * anything different based on what happens.
+ * We ignore pending signals since one will be set if the user is
+ * killed, and we need give the device a chance to cleanup, otherwise
+ * DMA may still be in progress when we return.
+ */
+ rsp = msg_xfer(qdev, wrapper, qdev->next_seq_num - 1, true);
+ if (!IS_ERR(rsp))
+ kfree(rsp);
+ kref_put(&wrapper->ref_count, free_wrapper);
+}
+
+void wake_all_cntl(struct qaic_device *qdev)
+{
+ struct xfer_queue_elem *elem;
+ struct xfer_queue_elem *i;
+
+ mutex_lock(&qdev->cntl_mutex);
+ list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
+ list_del_init(&elem->list);
+ complete_all(&elem->xfer_done);
+ }
+ mutex_unlock(&qdev->cntl_mutex);
+}
@@ -12,6 +12,11 @@
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
+#include <linux/sched.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/wait.h>
+#include <uapi/misc/qaic.h>
#include "mhi_controller.h"
#include "qaic.h"
@@ -21,6 +26,8 @@
#define QAIC_NAME "Qualcomm Cloud AI 100"
#define QAIC_MAX_MINORS 256
+static u16 cntl_major;
+static u16 cntl_minor = 3;
static int qaic_major;
static struct class *qaic_class;
static DEFINE_IDR(qaic_devs);
@@ -29,11 +36,14 @@ static bool link_up;
static int qaic_device_open(struct inode *inode, struct file *filp);
static int qaic_device_release(struct inode *inode, struct file *filp);
+static long qaic_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
static const struct file_operations qaic_ops = {
.owner = THIS_MODULE,
.open = qaic_device_open,
.release = qaic_device_release,
+ .unlocked_ioctl = qaic_ioctl,
+ .compat_ioctl = qaic_ioctl,
};
static void free_usr(struct kref *kref)
@@ -106,6 +116,7 @@ static int qaic_device_release(struct inode *inode, struct file *filp)
if (!qdev->in_reset) {
pci_dbg(qdev->pdev, "%s pid:%d\n", __func__,
current->pid);
+ qaic_release_usr(qdev, usr);
}
srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
@@ -123,12 +134,59 @@ static int qaic_device_release(struct inode *inode, struct file *filp)
return 0;
}
+static long qaic_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+ struct qaic_user *usr = filp->private_data;
+ struct qaic_device *qdev = usr->qdev;
+ unsigned int nr = _IOC_NR(cmd);
+ int qdev_rcu_id;
+ int usr_rcu_id;
+ int ret;
+
+ usr_rcu_id = srcu_read_lock(&usr->qdev_lock);
+ if (!qdev) {
+ srcu_read_unlock(&usr->qdev_lock, usr_rcu_id);
+ return -ENODEV;
+ }
+
+ qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
+ if (qdev->in_reset) {
+ srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+ srcu_read_unlock(&usr->qdev_lock, usr_rcu_id);
+ return -ENODEV;
+ }
+
+ if (_IOC_TYPE(cmd) != 'Q') {
+ srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+ srcu_read_unlock(&usr->qdev_lock, usr_rcu_id);
+ return -ENOTTY;
+ }
+
+ switch (nr) {
+ case QAIC_IOCTL_MANAGE_NR:
+ if (_IOC_DIR(cmd) != (_IOC_READ | _IOC_WRITE) ||
+ _IOC_SIZE(cmd) != sizeof(struct qaic_manage_msg)) {
+ ret = -EINVAL;
+ break;
+ }
+ ret = qaic_manage_ioctl(qdev, usr, arg);
+ break;
+ default:
+ ret = -ENOTTY;
+ }
+
+ srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
+ srcu_read_unlock(&usr->qdev_lock, usr_rcu_id);
+ return ret;
+}
+
static int qaic_mhi_probe(struct mhi_device *mhi_dev,
const struct mhi_device_id *id)
{
struct qaic_device *qdev;
dev_t devno;
int ret;
+ u16 major, minor;
/*
* Invoking this function indicates that the control channel to the
@@ -150,13 +208,26 @@ static int qaic_mhi_probe(struct mhi_device *mhi_dev,
dev_set_drvdata(&mhi_dev->dev, qdev);
qdev->cntl_ch = mhi_dev;
+ ret = qaic_control_open(qdev);
+ if (ret) {
+ pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
+ goto err;
+ }
+
+ ret = get_cntl_version(qdev, NULL, &major, &minor);
+ if (ret || major != cntl_major || minor > cntl_minor) {
+ pci_dbg(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n", __func__, major, minor, cntl_major, cntl_minor, ret);
+ ret = -EINVAL;
+ goto close_control;
+ }
+
mutex_lock(&qaic_devs_lock);
ret = idr_alloc(&qaic_devs, qdev, 0, QAIC_MAX_MINORS, GFP_KERNEL);
mutex_unlock(&qaic_devs_lock);
if (ret < 0) {
pci_dbg(qdev->pdev, "%s: idr_alloc failed %d\n", __func__, ret);
- goto err;
+ goto close_control;
}
devno = MKDEV(qaic_major, ret);
@@ -198,6 +269,8 @@ static int qaic_mhi_probe(struct mhi_device *mhi_dev,
mutex_lock(&qaic_devs_lock);
idr_remove(&qaic_devs, MINOR(devno));
mutex_unlock(&qaic_devs_lock);
+close_control:
+ qaic_control_close(qdev);
err:
return ret;
}
@@ -206,16 +279,6 @@ static void qaic_mhi_remove(struct mhi_device *mhi_dev)
{
}
-static void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev,
- struct mhi_result *mhi_result)
-{
-}
-
-static void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev,
- struct mhi_result *mhi_result)
-{
-}
-
void qaic_dev_reset_clean_local_state(struct qaic_device *qdev)
{
struct qaic_user *usr;
@@ -223,6 +286,8 @@ void qaic_dev_reset_clean_local_state(struct qaic_device *qdev)
dev_t devno;
qdev->in_reset = true;
+ /* wake up any waiters to avoid waiting for timeouts at sync */
+ wake_all_cntl(qdev);
synchronize_srcu(&qdev->dev_lock);
/*
@@ -255,10 +320,46 @@ void qaic_dev_reset_clean_local_state(struct qaic_device *qdev)
}
}
+inline int get_dbc_req_elem_size(void)
+{
+ return 64;
+}
+
+inline int get_dbc_rsp_elem_size(void)
+{
+ return 4;
+}
+
+int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr)
+{
+ if (!qdev->dbc[dbc_id].usr ||
+ qdev->dbc[dbc_id].usr->handle != usr->handle)
+ return -EPERM;
+
+ qdev->dbc[dbc_id].usr = NULL;
+ synchronize_srcu(&qdev->dbc[dbc_id].ch_lock);
+ return 0;
+}
+
+void release_dbc(struct qaic_device *qdev, u32 dbc_id)
+{
+ dma_free_coherent(&qdev->pdev->dev, qdev->dbc[dbc_id].total_size,
+ qdev->dbc[dbc_id].req_q_base,
+ qdev->dbc[dbc_id].dma_addr);
+ qdev->dbc[dbc_id].total_size = 0;
+ qdev->dbc[dbc_id].req_q_base = NULL;
+ qdev->dbc[dbc_id].dma_addr = 0;
+ qdev->dbc[dbc_id].nelem = 0;
+ qdev->dbc[dbc_id].usr = NULL;
+ qdev->dbc[dbc_id].in_use = false;
+ wake_up(&qdev->dbc[dbc_id].dbc_release);
+}
+
static int qaic_pci_probe(struct pci_dev *pdev,
const struct pci_device_id *id)
{
int ret;
+ int i;
int mhi_irq;
struct qaic_device *qdev;
@@ -270,11 +371,28 @@ static int qaic_pci_probe(struct pci_dev *pdev,
goto qdev_fail;
}
+ qdev->cntl_wq = alloc_workqueue("qaic_cntl", WQ_UNBOUND, 0);
+ if (!qdev->cntl_wq) {
+ ret = -ENOMEM;
+ goto wq_fail;
+ }
pci_set_drvdata(pdev, qdev);
qdev->pdev = pdev;
+ mutex_init(&qdev->cntl_mutex);
+ INIT_LIST_HEAD(&qdev->cntl_xfer_list);
init_srcu_struct(&qdev->dev_lock);
INIT_LIST_HEAD(&qdev->users);
mutex_init(&qdev->users_mutex);
+ for (i = 0; i < QAIC_NUM_DBC; ++i) {
+ mutex_init(&qdev->dbc[i].mem_lock);
+ spin_lock_init(&qdev->dbc[i].xfer_lock);
+ idr_init(&qdev->dbc[i].mem_handles);
+ qdev->dbc[i].qdev = qdev;
+ qdev->dbc[i].id = i;
+ INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
+ init_srcu_struct(&qdev->dbc[i].ch_lock);
+ init_waitqueue_head(&qdev->dbc[i].dbc_release);
+ }
qdev->bars = pci_select_bars(pdev, IORESOURCE_MEM);
@@ -308,6 +426,15 @@ static int qaic_pci_probe(struct pci_dev *pdev,
goto ioremap_0_fail;
}
+ qdev->bar_2 = pci_ioremap_bar(pdev, 2);
+ if (!qdev->bar_2) {
+ ret = -ENOMEM;
+ goto ioremap_2_fail;
+ }
+
+ for (i = 0; i < QAIC_NUM_DBC; ++i)
+ qdev->dbc[i].dbc_base = qdev->bar_2 + QAIC_DBC_OFF(i);
+
ret = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
if (ret < 0)
goto alloc_irq_fail;
@@ -339,6 +466,8 @@ static int qaic_pci_probe(struct pci_dev *pdev,
invalid_msi_config:
pci_free_irq_vectors(pdev);
alloc_irq_fail:
+ iounmap(qdev->bar_2);
+ioremap_2_fail:
iounmap(qdev->bar_0);
ioremap_0_fail:
dma_mask_fail:
@@ -349,7 +478,11 @@ static int qaic_pci_probe(struct pci_dev *pdev,
enable_fail:
pci_set_drvdata(pdev, NULL);
bar_fail:
+ for (i = 0; i < QAIC_NUM_DBC; ++i)
+ cleanup_srcu_struct(&qdev->dbc[i].ch_lock);
cleanup_srcu_struct(&qdev->dev_lock);
+ destroy_workqueue(qdev->cntl_wq);
+wq_fail:
kfree(qdev);
qdev_fail:
return ret;
@@ -358,6 +491,7 @@ static int qaic_pci_probe(struct pci_dev *pdev,
static void qaic_pci_remove(struct pci_dev *pdev)
{
struct qaic_device *qdev = pci_get_drvdata(pdev);
+ int i;
pci_dbg(pdev, "%s\n", __func__);
if (!qdev)
@@ -365,6 +499,9 @@ static void qaic_pci_remove(struct pci_dev *pdev)
qaic_dev_reset_clean_local_state(qdev);
qaic_mhi_free_controller(qdev->mhi_cntl, link_up);
+ for (i = 0; i < QAIC_NUM_DBC; ++i)
+ cleanup_srcu_struct(&qdev->dbc[i].ch_lock);
+ destroy_workqueue(qdev->cntl_wq);
pci_free_irq_vectors(pdev);
iounmap(qdev->bar_0);
pci_clear_master(pdev);
new file mode 100644
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+ *
+ * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef QAIC_H_
+#define QAIC_H_
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define QAIC_MANAGE_MAX_MSG_LENGTH 16364
+
+enum qaic_manage_transaction_type {
+ TRANS_UNDEFINED = 0,
+ TRANS_PASSTHROUGH_FROM_USR = 1,
+ TRANS_PASSTHROUGH_TO_USR = 2,
+ TRANS_PASSTHROUGH_FROM_DEV = 3,
+ TRANS_PASSTHROUGH_TO_DEV = 4,
+ TRANS_DMA_XFER_FROM_USR = 5,
+ TRANS_DMA_XFER_TO_DEV = 6,
+ TRANS_ACTIVATE_FROM_USR = 7,
+ TRANS_ACTIVATE_FROM_DEV = 8,
+ TRANS_ACTIVATE_TO_DEV = 9,
+ TRANS_DEACTIVATE_FROM_USR = 10,
+ TRANS_DEACTIVATE_FROM_DEV = 11,
+ TRANS_STATUS_FROM_USR = 12,
+ TRANS_STATUS_TO_USR = 13,
+ TRANS_STATUS_FROM_DEV = 14,
+ TRANS_STATUS_TO_DEV = 15,
+ TRANS_TERMINATE_FROM_DEV = 16,
+ TRANS_TERMINATE_TO_DEV = 17,
+ TRANS_MAX = 18
+};
+
+struct qaic_manage_trans_hdr {
+ __u32 type; /* value from enum manage_transaction_type */
+ __u32 len; /* length of this transaction, including the header */
+};
+
+struct qaic_manage_trans_passthrough {
+ struct qaic_manage_trans_hdr hdr;
+ u8 data[0]; /* userspace must encode in little endian */
+};
+
+struct qaic_manage_trans_dma_xfer {
+ struct qaic_manage_trans_hdr hdr;
+ __u32 tag;
+ __u32 count;
+ __u64 addr;
+ __u64 size;
+};
+
+struct qaic_manage_trans_activate_to_dev {
+ struct qaic_manage_trans_hdr hdr;
+ __u32 queue_size; /* in number of elements */
+ __u32 eventfd;
+ __u64 resv; /* reserved for future use, must be 0 */
+};
+
+struct qaic_manage_trans_activate_from_dev {
+ struct qaic_manage_trans_hdr hdr;
+ __u32 status;
+ __u32 dbc_id; /* Identifier of assigned DMA Bridge channel */
+};
+
+struct qaic_manage_trans_deactivate {
+ struct qaic_manage_trans_hdr hdr;
+ __u32 dbc_id; /* Identifier of assigned DMA Bridge channel */
+ __u32 resv; /* reserved for future use, must be 0 */
+};
+
+struct qaic_manage_trans_status_to_dev {
+ struct qaic_manage_trans_hdr hdr;
+};
+
+struct qaic_manage_trans_status_from_dev {
+ struct qaic_manage_trans_hdr hdr;
+ __u16 major;
+ __u16 minor;
+ __u32 status;
+};
+
+struct qaic_manage_msg {
+ __u32 len; /* Length of valid data - ie sum of all transactions */
+ __u32 count; /* Number of transactions in message */
+ __u8 data[QAIC_MANAGE_MAX_MSG_LENGTH];
+};
+
+#define QAIC_IOCTL_MANAGE_NR 0x01
+
+/*
+ * Send Manage command to the device
+ *
+ * A manage command is a message that consists of N transactions. The set
+ * of transactions consititues a single operation. In most cases, a manage
+ * command is a request for the device to do something. The entire command
+ * must be encoded into a single message.
+ *
+ * The command will be encoded into the wire format, and sent to the device.
+ * the process will then be blocked until the device responds to the message
+ * or a timeout is reached. If a response is successfully received, it will
+ * be encoded into the provided message structure.
+ *
+ * The return value is 0 for success, or a standard error code. Some of the
+ * possible errors:
+ *
+ * EINTR - Kernel waiting was interrupted (IE received a signal for user)
+ * ETIMEDOUT - Timeout for response from device expired
+ * EINVAL - Invalid message
+ * ENOSPC - Ran out of space to encode the message into the wire protocol
+ * ENOMEM - Unable to obtain memory while processing message
+ * EFAULT - Error in accessing memory from user
+ */
+#define QAIC_IOCTL_MANAGE _IOWR('Q', QAIC_IOCTL_MANAGE_NR, struct manage_msg)
+
+#endif /* QAIC_H_ */
Once a qaic device is fully booted, the control path is available for use. The control path allows the host to use a MHI channel to send requests to the device. The canonical usecase for the control path is allowing userspace to configure the device for a workload. This is accomplished via the manage ioctl. Signed-off-by: Jeffrey Hugo <jhugo@codeaurora.org> --- drivers/misc/qaic/Makefile | 3 +- drivers/misc/qaic/qaic.h | 61 +++ drivers/misc/qaic/qaic_control.c | 1015 ++++++++++++++++++++++++++++++++++++++ drivers/misc/qaic/qaic_drv.c | 159 +++++- include/uapi/misc/qaic.h | 117 +++++ 5 files changed, 1343 insertions(+), 12 deletions(-) create mode 100644 drivers/misc/qaic/qaic_control.c create mode 100644 include/uapi/misc/qaic.h