@@ -3166,6 +3166,7 @@ static int nvme_dev_open(struct inode *inode, struct file *file)
{
struct nvme_ctrl *ctrl =
container_of(inode->i_cdev, struct nvme_ctrl, cdev);
+ int ret = -EINVAL;
switch (ctrl->state) {
case NVME_CTRL_LIVE:
@@ -3175,13 +3176,25 @@ static int nvme_dev_open(struct inode *inode, struct file *file)
}
nvme_get_ctrl(ctrl);
- if (!try_module_get(ctrl->ops->module)) {
- nvme_put_ctrl(ctrl);
- return -EINVAL;
- }
+ if (!try_module_get(ctrl->ops->module))
+ goto err_put_ctrl;
file->private_data = ctrl;
+
+ if (ctrl->ops->cdev_file_open) {
+ ret = ctrl->ops->cdev_file_open(ctrl, file);
+ if (ret)
+ goto err_put_mod;
+ }
+
return 0;
+
+err_put_mod:
+ module_put(ctrl->ops->module);
+err_put_ctrl:
+ nvme_put_ctrl(ctrl);
+ return ret;
+
}
static int nvme_dev_release(struct inode *inode, struct file *file)
@@ -3189,11 +3202,24 @@ static int nvme_dev_release(struct inode *inode, struct file *file)
struct nvme_ctrl *ctrl =
container_of(inode->i_cdev, struct nvme_ctrl, cdev);
+ if (ctrl->ops->cdev_file_release)
+ ctrl->ops->cdev_file_release(file);
+
module_put(ctrl->ops->module);
nvme_put_ctrl(ctrl);
return 0;
}
+static int nvme_dev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct nvme_ctrl *ctrl = file->private_data;
+
+ if (!ctrl->ops->mmap_cmb)
+ return -ENODEV;
+
+ return ctrl->ops->mmap_cmb(ctrl, vma);
+}
+
static const struct file_operations nvme_dev_fops = {
.owner = THIS_MODULE,
.open = nvme_dev_open,
@@ -3201,6 +3227,7 @@ static const struct file_operations nvme_dev_fops = {
.unlocked_ioctl = nvme_dev_ioctl,
.compat_ioctl = compat_ptr_ioctl,
.uring_cmd = nvme_dev_uring_cmd,
+ .mmap = nvme_dev_mmap,
};
static ssize_t nvme_sysfs_reset(struct device *dev,
@@ -503,6 +503,9 @@ struct nvme_ctrl_ops {
void (*delete_ctrl)(struct nvme_ctrl *ctrl);
int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl);
+ int (*cdev_file_open)(struct nvme_ctrl *ctrl, struct file *file);
+ void (*cdev_file_release)(struct file *file);
+ int (*mmap_cmb)(struct nvme_ctrl *ctrl, struct vm_area_struct *vma);
};
/*
@@ -2972,6 +2972,26 @@ static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl)
return dma_pci_p2pdma_supported(dev->dev);
}
+static int nvme_pci_cdev_file_open(struct nvme_ctrl *ctrl, struct file *file)
+{
+ struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
+
+ return pci_p2pdma_file_open(pdev, file);
+}
+
+static void nvme_pci_cdev_file_release(struct file *file)
+{
+ pci_p2pdma_file_release(file);
+}
+
+static int nvme_pci_mmap_cmb(struct nvme_ctrl *ctrl,
+ struct vm_area_struct *vma)
+{
+ struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
+
+ return pci_mmap_p2pmem(pdev, vma);
+}
+
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.name = "pcie",
.module = THIS_MODULE,
@@ -2983,6 +3003,9 @@ static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
.submit_async_event = nvme_pci_submit_async_event,
.get_address = nvme_pci_get_address,
.supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma,
+ .cdev_file_open = nvme_pci_cdev_file_open,
+ .cdev_file_release = nvme_pci_cdev_file_release,
+ .mmap_cmb = nvme_pci_mmap_cmb,
};
static int nvme_dev_map(struct nvme_dev *dev)
Allow userspace to obtain CMB memory by mmaping the controller's char device. The mmap call allocates and returns a hunk of CMB memory, (the offset is ignored) so userspace does not have control over the address within the CMB. A VMA allocated in this way will only be usable by drivers that set FOLL_PCI_P2PDMA when calling GUP. And inter-device support will be checked the first time the pages are mapped for DMA. Currently this is only supported by O_DIRECT to an PCI NVMe device or through the NVMe passthrough IOCTL. Signed-off-by: Logan Gunthorpe <logang@deltatee.com> --- drivers/nvme/host/core.c | 35 +++++++++++++++++++++++++++++++---- drivers/nvme/host/nvme.h | 3 +++ drivers/nvme/host/pci.c | 23 +++++++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-)