@@ -4,6 +4,7 @@ config VFIO_PCI
depends on VFIO && PCI && EVENTFD
select VFIO_VIRQFD
select IRQ_BYPASS_MANAGER
+ select SRCU
help
Support for the PCI VFIO bus driver. This is required to make
use of PCI drivers using the VFIO framework.
@@ -560,6 +560,8 @@ static void vfio_pci_release(void *device_data)
mutex_lock(&vdev->reflck->lock);
if (!(--vdev->refcnt)) {
+ srcu_notifier_call_chain(&vdev->vma_notifier,
+ VFIO_VMA_NOTIFY_CLOSE, NULL);
vfio_pci_vf_token_user_add(vdev, -1);
vfio_spapr_pci_eeh_release(vdev->pdev);
vfio_pci_disable(vdev);
@@ -1969,6 +1971,7 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
mutex_init(&vdev->vma_lock);
INIT_LIST_HEAD(&vdev->vma_list);
init_rwsem(&vdev->memory_lock);
+ srcu_init_notifier_head(&vdev->vma_notifier);
ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
if (ret)
@@ -2362,6 +2365,7 @@ static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
static void __exit vfio_pci_cleanup(void)
{
+ vfio_unregister_vma_ops(&vfio_pci_mmap_ops);
pci_unregister_driver(&vfio_pci_driver);
vfio_pci_uninit_perm_bits();
}
@@ -2407,6 +2411,81 @@ static void __init vfio_pci_fill_ids(void)
}
}
+struct vfio_pci_vma_obj {
+ struct vfio_pci_device *vdev;
+ struct vfio_group *group;
+ struct vfio_device *device;
+ struct notifier_block *nb;
+};
+
+static void *vfio_pci_register_vma_notifier(struct vm_area_struct *vma,
+ struct notifier_block *nb)
+{
+ struct vfio_pci_device *vdev = vma->vm_private_data;
+ struct vfio_pci_vma_obj *obj;
+ struct vfio_group *group;
+ struct vfio_device *device;
+ int ret;
+
+ if (!vdev || vma->vm_ops != &vfio_pci_mmap_ops)
+ return ERR_PTR(-EINVAL);
+
+ obj = kmalloc(sizeof(*obj), GFP_KERNEL);
+ if (!obj)
+ return ERR_PTR(-ENOMEM);
+
+ /*
+ * Get a group and container reference, this prevents the container
+ * from being torn down while this vma is mapped, ie. device stays
+ * isolated.
+ *
+ * NB. The container must be torn down on device close without
+ * explicit unmaps, therefore we must notify on close.
+ */
+ group = vfio_group_get_external_user_from_dev(&vdev->pdev->dev);
+ if (IS_ERR(group)) {
+ kfree(obj);
+ return group;
+ }
+
+ /* Also need device reference to prevent unbind */
+ device = vfio_device_get_from_dev(&vdev->pdev->dev);
+ if (IS_ERR(device)) {
+ vfio_group_put_external_user(group);
+ kfree(obj);
+ return device;
+ }
+
+ /*
+ * Use the srcu notifier chain variant to avoid AB-BA locking issues
+ * with the caller, ex. iommu->lock vs nh->rwsem
+ */
+ ret = srcu_notifier_chain_register(&vdev->vma_notifier, nb);
+ if (ret) {
+ vfio_device_put(device);
+ vfio_group_put_external_user(group);
+ kfree(obj);
+ return ERR_PTR(ret);
+ }
+
+ obj->vdev = vdev;
+ obj->group = group;
+ obj->device = device;
+ obj->nb = nb;
+
+ return obj;
+}
+
+static void vfio_pci_unregister_vma_notifier(void *opaque)
+{
+ struct vfio_pci_vma_obj *obj = opaque;
+
+ srcu_notifier_chain_unregister(&obj->vdev->vma_notifier, obj->nb);
+ vfio_device_put(obj->device);
+ vfio_group_put_external_user(obj->group);
+ kfree(obj);
+}
+
static int __init vfio_pci_init(void)
{
int ret;
@@ -2421,6 +2500,12 @@ static int __init vfio_pci_init(void)
if (ret)
goto out_driver;
+ ret = vfio_register_vma_ops(&vfio_pci_mmap_ops,
+ vfio_pci_register_vma_notifier,
+ vfio_pci_unregister_vma_notifier);
+ if (ret)
+ goto out_vma;
+
vfio_pci_fill_ids();
if (disable_denylist)
@@ -2428,6 +2513,8 @@ static int __init vfio_pci_init(void)
return 0;
+out_vma:
+ pci_unregister_driver(&vfio_pci_driver);
out_driver:
vfio_pci_uninit_perm_bits();
return ret;
@@ -142,6 +142,7 @@ struct vfio_pci_device {
struct mutex vma_lock;
struct list_head vma_list;
struct rw_semaphore memory_lock;
+ struct srcu_notifier_head vma_notifier;
};
#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
The vfio-pci vfio bus driver implements a vm_operations_struct for managing mmaps to device BARs, therefore given a vma with matching vm_ops we can create a reference using the existing vfio external user interfaces and register the provided notifier to receive callbacks relative to the device. The close notifier is implemented for when the device is released, rather than closing the vma to avoid possibly breaking userspace (ie. mmap -> dma map -> munmap is currently allowed and maintains the dma mapping to the device). Signed-off-by: Alex Williamson <alex.williamson@redhat.com> --- drivers/vfio/pci/Kconfig | 1 drivers/vfio/pci/vfio_pci.c | 87 +++++++++++++++++++++++++++++++++++ drivers/vfio/pci/vfio_pci_private.h | 1 3 files changed, 89 insertions(+)