| Message ID | 20221125023045.2158413-1-lulu@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | [v3] vhost_vdpa: fix the crash in unmap a large memory | expand |
On Fri, Nov 25, 2022 at 10:31 AM Cindy Lu <lulu@redhat.com> wrote: > > While testing in vIOMMU, sometimes guest will unmap very large memory, > which will cause the crash. To fix this,Move the iommu_unmap to > vhost_vdpa_pa_unmap/vhost_vdpa_va_unmap and only unmap the memory > that saved in iotlb. > > Call Trace: > [ 647.820144] ------------[ cut here ]------------ > [ 647.820848] kernel BUG at drivers/iommu/intel/iommu.c:1174! > [ 647.821486] invalid opcode: 0000 [#1] PREEMPT SMP PTI > [ 647.822082] CPU: 10 PID: 1181 Comm: qemu-system-x86 Not tainted 6.0.0-rc1home_lulu_2452_lulu7_vhost+ #62 > [ 647.823139] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.15.0-29-g6a62e0cb0dfe-prebuilt.qem4 > [ 647.824365] RIP: 0010:domain_unmap+0x48/0x110 > [ 647.825424] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > [ 647.828064] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > [ 647.828973] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > [ 647.830083] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > [ 647.831214] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > [ 647.832388] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > [ 647.833668] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > [ 647.834782] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > [ 647.836004] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > [ 647.836990] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > [ 647.838107] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > [ 647.839283] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 > [ 647.840666] Call Trace: > [ 647.841437] <TASK> > [ 647.842107] intel_iommu_unmap_pages+0x93/0x140 > [ 647.843112] __iommu_unmap+0x91/0x1b0 > [ 647.844003] iommu_unmap+0x6a/0x95 > [ 647.844885] vhost_vdpa_unmap+0x1de/0x1f0 [vhost_vdpa] > [ 647.845985] vhost_vdpa_process_iotlb_msg+0xf0/0x90b [vhost_vdpa] > [ 647.847235] ? _raw_spin_unlock+0x15/0x30 > [ 647.848181] ? _copy_from_iter+0x8c/0x580 > [ 647.849137] vhost_chr_write_iter+0xb3/0x430 [vhost] > [ 647.850126] vfs_write+0x1e4/0x3a0 > [ 647.850897] ksys_write+0x53/0xd0 > [ 647.851688] do_syscall_64+0x3a/0x90 > [ 647.852508] entry_SYSCALL_64_after_hwframe+0x63/0xcd > [ 647.853457] RIP: 0033:0x7f7734ef9f4f > [ 647.854408] Code: 89 54 24 18 48 89 74 24 10 89 7c 24 08 e8 29 76 f8 ff 48 8b 54 24 18 48 8b 74 24 10 41 89 c8 > [ 647.857217] RSP: 002b:00007f772ec8f040 EFLAGS: 00000293 ORIG_RAX: 0000000000000001 > [ 647.858486] RAX: ffffffffffffffda RBX: 00000000fef00000 RCX: 00007f7734ef9f4f > [ 647.859713] RDX: 0000000000000048 RSI: 00007f772ec8f090 RDI: 0000000000000010 > [ 647.860942] RBP: 00007f772ec8f1a0 R08: 0000000000000000 R09: 0000000000000000 > [ 647.862206] R10: 0000000000000001 R11: 0000000000000293 R12: 0000000000000010 > [ 647.863446] R13: 0000000000000002 R14: 0000000000000000 R15: ffffffff01100000 > [ 647.864692] </TASK> > [ 647.865458] Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache netfs v] > [ 647.874688] ---[ end trace 0000000000000000 ]--- > [ 647.876013] RIP: 0010:domain_unmap+0x48/0x110 > [ 647.878306] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > [ 647.884581] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > [ 647.886308] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > [ 647.888775] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > [ 647.890295] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > [ 647.891660] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > [ 647.893019] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > [ 647.894506] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > [ 647.895963] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > [ 647.897348] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > [ 647.898719] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > > Cc: stable@vger.kernel.org > Fixes: 4c8cf31885f6 ("vhost: introduce vDPA-based backend") > Signed-off-by: Cindy Lu <lulu@redhat.com> > --- > drivers/vhost/vdpa.c | 10 ++++++++-- > 1 file changed, 8 insertions(+), 2 deletions(-) > > diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c > index 166044642fd5..e5a07751bf45 100644 > --- a/drivers/vhost/vdpa.c > +++ b/drivers/vhost/vdpa.c > @@ -692,6 +692,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > struct vhost_iotlb_map *map; > struct page *page; > unsigned long pfn, pinned; > + struct vdpa_device *vdpa = v->vdpa; > + const struct vdpa_config_ops *ops = vdpa->config; > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > pinned = PFN_DOWN(map->size); > @@ -703,6 +705,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > unpin_user_page(page); > } > atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); > + if ((ops->dma_map == NULL) && (ops->set_map == NULL)) > + iommu_unmap(v->domain, map->start, map->size); I think we'd better move the ops->dma_unmap() here as well as iommu_unmap()? > vhost_iotlb_map_free(iotlb, map); > } > } > @@ -713,11 +717,15 @@ static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, > { > struct vhost_iotlb_map *map; > struct vdpa_map_file *map_file; > + struct vdpa_device *vdpa = v->vdpa; > + const struct vdpa_config_ops *ops = vdpa->config; > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > map_file = (struct vdpa_map_file *)map->opaque; > fput(map_file->file); > kfree(map_file); > + if (ops->set_map == NULL) > + iommu_unmap(v->domain, map->start, map->size); Need to check where we have dma_unmap() and call that if it exists? Thanks > vhost_iotlb_map_free(iotlb, map); > } > } > @@ -805,8 +813,6 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v, > } else if (ops->set_map) { > if (!v->in_batch) > ops->set_map(vdpa, asid, iotlb); > - } else { > - iommu_unmap(v->domain, iova, size); > } > > /* If we are in the middle of batch processing, delay the free > -- > 2.34.3 >
/ and On Fri, 25 Nov 2022 at 15:17, Jason Wang <jasowang@redhat.com> wrote: > > On Fri, Nov 25, 2022 at 10:31 AM Cindy Lu <lulu@redhat.com> wrote: > > > > While testing in vIOMMU, sometimes guest will unmap very large memory, > > which will cause the crash. To fix this,Move the iommu_unmap to > > vhost_vdpa_pa_unmap/vhost_vdpa_va_unmap and only unmap the memory > > that saved in iotlb. > > > > Call Trace: > > [ 647.820144] ------------[ cut here ]------------ > > [ 647.820848] kernel BUG at drivers/iommu/intel/iommu.c:1174! > > [ 647.821486] invalid opcode: 0000 [#1] PREEMPT SMP PTI > > [ 647.822082] CPU: 10 PID: 1181 Comm: qemu-system-x86 Not tainted 6.0.0-rc1home_lulu_2452_lulu7_vhost+ #62 > > [ 647.823139] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.15.0-29-g6a62e0cb0dfe-prebuilt.qem4 > > [ 647.824365] RIP: 0010:domain_unmap+0x48/0x110 > > [ 647.825424] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > > [ 647.828064] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > > [ 647.828973] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > > [ 647.830083] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > > [ 647.831214] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > > [ 647.832388] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > > [ 647.833668] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > > [ 647.834782] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > > [ 647.836004] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > > [ 647.836990] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > > [ 647.838107] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > > [ 647.839283] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 > > [ 647.840666] Call Trace: > > [ 647.841437] <TASK> > > [ 647.842107] intel_iommu_unmap_pages+0x93/0x140 > > [ 647.843112] __iommu_unmap+0x91/0x1b0 > > [ 647.844003] iommu_unmap+0x6a/0x95 > > [ 647.844885] vhost_vdpa_unmap+0x1de/0x1f0 [vhost_vdpa] > > [ 647.845985] vhost_vdpa_process_iotlb_msg+0xf0/0x90b [vhost_vdpa] > > [ 647.847235] ? _raw_spin_unlock+0x15/0x30 > > [ 647.848181] ? _copy_from_iter+0x8c/0x580 > > [ 647.849137] vhost_chr_write_iter+0xb3/0x430 [vhost] > > [ 647.850126] vfs_write+0x1e4/0x3a0 > > [ 647.850897] ksys_write+0x53/0xd0 > > [ 647.851688] do_syscall_64+0x3a/0x90 > > [ 647.852508] entry_SYSCALL_64_after_hwframe+0x63/0xcd > > [ 647.853457] RIP: 0033:0x7f7734ef9f4f > > [ 647.854408] Code: 89 54 24 18 48 89 74 24 10 89 7c 24 08 e8 29 76 f8 ff 48 8b 54 24 18 48 8b 74 24 10 41 89 c8 > > [ 647.857217] RSP: 002b:00007f772ec8f040 EFLAGS: 00000293 ORIG_RAX: 0000000000000001 > > [ 647.858486] RAX: ffffffffffffffda RBX: 00000000fef00000 RCX: 00007f7734ef9f4f > > [ 647.859713] RDX: 0000000000000048 RSI: 00007f772ec8f090 RDI: 0000000000000010 > > [ 647.860942] RBP: 00007f772ec8f1a0 R08: 0000000000000000 R09: 0000000000000000 > > [ 647.862206] R10: 0000000000000001 R11: 0000000000000293 R12: 0000000000000010 > > [ 647.863446] R13: 0000000000000002 R14: 0000000000000000 R15: ffffffff01100000 > > [ 647.864692] </TASK> > > [ 647.865458] Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache netfs v] > > [ 647.874688] ---[ end trace 0000000000000000 ]--- > > [ 647.876013] RIP: 0010:domain_unmap+0x48/0x110 > > [ 647.878306] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > > [ 647.884581] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > > [ 647.886308] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > > [ 647.888775] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > > [ 647.890295] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > > [ 647.891660] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > > [ 647.893019] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > > [ 647.894506] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > > [ 647.895963] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > > [ 647.897348] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > > [ 647.898719] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > > > > Cc: stable@vger.kernel.org > > Fixes: 4c8cf31885f6 ("vhost: introduce vDPA-based backend") > > Signed-off-by: Cindy Lu <lulu@redhat.com> > > --- > > drivers/vhost/vdpa.c | 10 ++++++++-- > > 1 file changed, 8 insertions(+), 2 deletions(-) > > > > diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c > > index 166044642fd5..e5a07751bf45 100644 > > --- a/drivers/vhost/vdpa.c > > +++ b/drivers/vhost/vdpa.c > > @@ -692,6 +692,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > > struct vhost_iotlb_map *map; > > struct page *page; > > unsigned long pfn, pinned; > > + struct vdpa_device *vdpa = v->vdpa; > > + const struct vdpa_config_ops *ops = vdpa->config; > > > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > > pinned = PFN_DOWN(map->size); > > @@ -703,6 +705,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > > unpin_user_page(page); > > } > > atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); > > + if ((ops->dma_map == NULL) && (ops->set_map == NULL)) > > + iommu_unmap(v->domain, map->start, map->size); > > I think we'd better move the ops->dma_unmap() here as well as iommu_unmap()? > > > vhost_iotlb_map_free(iotlb, map); > > } > > } > > @@ -713,11 +717,15 @@ static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, > > { > > struct vhost_iotlb_map *map; > > struct vdpa_map_file *map_file; > > + struct vdpa_device *vdpa = v->vdpa; > > + const struct vdpa_config_ops *ops = vdpa->config; > > > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > > map_file = (struct vdpa_map_file *)map->opaque; > > fput(map_file->file); > > kfree(map_file); > > + if (ops->set_map == NULL) > > + iommu_unmap(v->domain, map->start, map->size); > > Need to check where we have dma_unmap() and call that if it exists? > > Thanks > Hi Jason， I think these functions are called in vhost_vdpa_unmap, Do you want to separate the function in vhost_vdpa_unmap and move it to vhost_vdpa_va_unmap and vhost_vdpa_pa_unmap? I thanks cindy > > vhost_iotlb_map_free(iotlb, map); > > } > > } > > @@ -805,8 +813,6 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v, > > } else if (ops->set_map) { > > if (!v->in_batch) > > ops->set_map(vdpa, asid, iotlb); > > - } else { > > - iommu_unmap(v->domain, iova, size); > > } > > > > /* If we are in the middle of batch processing, delay the free > > -- > > 2.34.3 > > >
On Fri, Nov 25, 2022 at 3:38 PM Cindy Lu <lulu@redhat.com> wrote: > > / and > > > On Fri, 25 Nov 2022 at 15:17, Jason Wang <jasowang@redhat.com> wrote: > > > > On Fri, Nov 25, 2022 at 10:31 AM Cindy Lu <lulu@redhat.com> wrote: > > > > > > While testing in vIOMMU, sometimes guest will unmap very large memory, > > > which will cause the crash. To fix this,Move the iommu_unmap to > > > vhost_vdpa_pa_unmap/vhost_vdpa_va_unmap and only unmap the memory > > > that saved in iotlb. > > > > > > Call Trace: > > > [ 647.820144] ------------[ cut here ]------------ > > > [ 647.820848] kernel BUG at drivers/iommu/intel/iommu.c:1174! > > > [ 647.821486] invalid opcode: 0000 [#1] PREEMPT SMP PTI > > > [ 647.822082] CPU: 10 PID: 1181 Comm: qemu-system-x86 Not tainted 6.0.0-rc1home_lulu_2452_lulu7_vhost+ #62 > > > [ 647.823139] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.15.0-29-g6a62e0cb0dfe-prebuilt.qem4 > > > [ 647.824365] RIP: 0010:domain_unmap+0x48/0x110 > > > [ 647.825424] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > > > [ 647.828064] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > > > [ 647.828973] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > > > [ 647.830083] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > > > [ 647.831214] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > > > [ 647.832388] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > > > [ 647.833668] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > > > [ 647.834782] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > > > [ 647.836004] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > > > [ 647.836990] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > > > [ 647.838107] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > > > [ 647.839283] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 > > > [ 647.840666] Call Trace: > > > [ 647.841437] <TASK> > > > [ 647.842107] intel_iommu_unmap_pages+0x93/0x140 > > > [ 647.843112] __iommu_unmap+0x91/0x1b0 > > > [ 647.844003] iommu_unmap+0x6a/0x95 > > > [ 647.844885] vhost_vdpa_unmap+0x1de/0x1f0 [vhost_vdpa] > > > [ 647.845985] vhost_vdpa_process_iotlb_msg+0xf0/0x90b [vhost_vdpa] > > > [ 647.847235] ? _raw_spin_unlock+0x15/0x30 > > > [ 647.848181] ? _copy_from_iter+0x8c/0x580 > > > [ 647.849137] vhost_chr_write_iter+0xb3/0x430 [vhost] > > > [ 647.850126] vfs_write+0x1e4/0x3a0 > > > [ 647.850897] ksys_write+0x53/0xd0 > > > [ 647.851688] do_syscall_64+0x3a/0x90 > > > [ 647.852508] entry_SYSCALL_64_after_hwframe+0x63/0xcd > > > [ 647.853457] RIP: 0033:0x7f7734ef9f4f > > > [ 647.854408] Code: 89 54 24 18 48 89 74 24 10 89 7c 24 08 e8 29 76 f8 ff 48 8b 54 24 18 48 8b 74 24 10 41 89 c8 > > > [ 647.857217] RSP: 002b:00007f772ec8f040 EFLAGS: 00000293 ORIG_RAX: 0000000000000001 > > > [ 647.858486] RAX: ffffffffffffffda RBX: 00000000fef00000 RCX: 00007f7734ef9f4f > > > [ 647.859713] RDX: 0000000000000048 RSI: 00007f772ec8f090 RDI: 0000000000000010 > > > [ 647.860942] RBP: 00007f772ec8f1a0 R08: 0000000000000000 R09: 0000000000000000 > > > [ 647.862206] R10: 0000000000000001 R11: 0000000000000293 R12: 0000000000000010 > > > [ 647.863446] R13: 0000000000000002 R14: 0000000000000000 R15: ffffffff01100000 > > > [ 647.864692] </TASK> > > > [ 647.865458] Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache netfs v] > > > [ 647.874688] ---[ end trace 0000000000000000 ]--- > > > [ 647.876013] RIP: 0010:domain_unmap+0x48/0x110 > > > [ 647.878306] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > > > [ 647.884581] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > > > [ 647.886308] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > > > [ 647.888775] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > > > [ 647.890295] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > > > [ 647.891660] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > > > [ 647.893019] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > > > [ 647.894506] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > > > [ 647.895963] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > > > [ 647.897348] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > > > [ 647.898719] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > > > > > > Cc: stable@vger.kernel.org > > > Fixes: 4c8cf31885f6 ("vhost: introduce vDPA-based backend") > > > Signed-off-by: Cindy Lu <lulu@redhat.com> > > > --- > > > drivers/vhost/vdpa.c | 10 ++++++++-- > > > 1 file changed, 8 insertions(+), 2 deletions(-) > > > > > > diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c > > > index 166044642fd5..e5a07751bf45 100644 > > > --- a/drivers/vhost/vdpa.c > > > +++ b/drivers/vhost/vdpa.c > > > @@ -692,6 +692,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > > > struct vhost_iotlb_map *map; > > > struct page *page; > > > unsigned long pfn, pinned; > > > + struct vdpa_device *vdpa = v->vdpa; > > > + const struct vdpa_config_ops *ops = vdpa->config; > > > > > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > > > pinned = PFN_DOWN(map->size); > > > @@ -703,6 +705,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > > > unpin_user_page(page); > > > } > > > atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); > > > + if ((ops->dma_map == NULL) && (ops->set_map == NULL)) > > > + iommu_unmap(v->domain, map->start, map->size); > > > > I think we'd better move the ops->dma_unmap() here as well as iommu_unmap()? > > > > > vhost_iotlb_map_free(iotlb, map); > > > } > > > } > > > @@ -713,11 +717,15 @@ static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, > > > { > > > struct vhost_iotlb_map *map; > > > struct vdpa_map_file *map_file; > > > + struct vdpa_device *vdpa = v->vdpa; > > > + const struct vdpa_config_ops *ops = vdpa->config; > > > > > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > > > map_file = (struct vdpa_map_file *)map->opaque; > > > fput(map_file->file); > > > kfree(map_file); > > > + if (ops->set_map == NULL) > > > + iommu_unmap(v->domain, map->start, map->size); > > > > Need to check where we have dma_unmap() and call that if it exists? > > > > Thanks > > > Hi Jason， > I think these functions are called in vhost_vdpa_unmap, > Do you want to separate the function in vhost_vdpa_unmap > and move it to vhost_vdpa_va_unmap and vhost_vdpa_pa_unmap? I I meant dma_map()/dma_unmap() should be functional equivalent to iommu_map/unmap(). That means we should unmap exactly what is mapped before (vDPA parent may call iommu_unmap in its own dma_unmap() if it needs). If we move the iommu_unmap() from vhost_vdpa_unmap() to vhost_vdpa_{pa|va}_umap, we should move dma_unmap() as well. Thanks > thanks > cindy > > > > vhost_iotlb_map_free(iotlb, map); > > > } > > > } > > > @@ -805,8 +813,6 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v, > > > } else if (ops->set_map) { > > > if (!v->in_batch) > > > ops->set_map(vdpa, asid, iotlb); > > > - } else { > > > - iommu_unmap(v->domain, iova, size); > > > } > > > > > > /* If we are in the middle of batch processing, delay the free > > > -- > > > 2.34.3 > > > > > >
On Tue, 29 Nov 2022 at 11:04, Jason Wang <jasowang@redhat.com> wrote: > > On Fri, Nov 25, 2022 at 3:38 PM Cindy Lu <lulu@redhat.com> wrote: > > > > / and > > > > > > On Fri, 25 Nov 2022 at 15:17, Jason Wang <jasowang@redhat.com> wrote: > > > > > > On Fri, Nov 25, 2022 at 10:31 AM Cindy Lu <lulu@redhat.com> wrote: > > > > > > > > While testing in vIOMMU, sometimes guest will unmap very large memory, > > > > which will cause the crash. To fix this,Move the iommu_unmap to > > > > vhost_vdpa_pa_unmap/vhost_vdpa_va_unmap and only unmap the memory > > > > that saved in iotlb. > > > > > > > > Call Trace: > > > > [ 647.820144] ------------[ cut here ]------------ > > > > [ 647.820848] kernel BUG at drivers/iommu/intel/iommu.c:1174! > > > > [ 647.821486] invalid opcode: 0000 [#1] PREEMPT SMP PTI > > > > [ 647.822082] CPU: 10 PID: 1181 Comm: qemu-system-x86 Not tainted 6.0.0-rc1home_lulu_2452_lulu7_vhost+ #62 > > > > [ 647.823139] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.15.0-29-g6a62e0cb0dfe-prebuilt.qem4 > > > > [ 647.824365] RIP: 0010:domain_unmap+0x48/0x110 > > > > [ 647.825424] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > > > > [ 647.828064] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > > > > [ 647.828973] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > > > > [ 647.830083] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > > > > [ 647.831214] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > > > > [ 647.832388] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > > > > [ 647.833668] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > > > > [ 647.834782] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > > > > [ 647.836004] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > > > > [ 647.836990] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > > > > [ 647.838107] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > > > > [ 647.839283] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 > > > > [ 647.840666] Call Trace: > > > > [ 647.841437] <TASK> > > > > [ 647.842107] intel_iommu_unmap_pages+0x93/0x140 > > > > [ 647.843112] __iommu_unmap+0x91/0x1b0 > > > > [ 647.844003] iommu_unmap+0x6a/0x95 > > > > [ 647.844885] vhost_vdpa_unmap+0x1de/0x1f0 [vhost_vdpa] > > > > [ 647.845985] vhost_vdpa_process_iotlb_msg+0xf0/0x90b [vhost_vdpa] > > > > [ 647.847235] ? _raw_spin_unlock+0x15/0x30 > > > > [ 647.848181] ? _copy_from_iter+0x8c/0x580 > > > > [ 647.849137] vhost_chr_write_iter+0xb3/0x430 [vhost] > > > > [ 647.850126] vfs_write+0x1e4/0x3a0 > > > > [ 647.850897] ksys_write+0x53/0xd0 > > > > [ 647.851688] do_syscall_64+0x3a/0x90 > > > > [ 647.852508] entry_SYSCALL_64_after_hwframe+0x63/0xcd > > > > [ 647.853457] RIP: 0033:0x7f7734ef9f4f > > > > [ 647.854408] Code: 89 54 24 18 48 89 74 24 10 89 7c 24 08 e8 29 76 f8 ff 48 8b 54 24 18 48 8b 74 24 10 41 89 c8 > > > > [ 647.857217] RSP: 002b:00007f772ec8f040 EFLAGS: 00000293 ORIG_RAX: 0000000000000001 > > > > [ 647.858486] RAX: ffffffffffffffda RBX: 00000000fef00000 RCX: 00007f7734ef9f4f > > > > [ 647.859713] RDX: 0000000000000048 RSI: 00007f772ec8f090 RDI: 0000000000000010 > > > > [ 647.860942] RBP: 00007f772ec8f1a0 R08: 0000000000000000 R09: 0000000000000000 > > > > [ 647.862206] R10: 0000000000000001 R11: 0000000000000293 R12: 0000000000000010 > > > > [ 647.863446] R13: 0000000000000002 R14: 0000000000000000 R15: ffffffff01100000 > > > > [ 647.864692] </TASK> > > > > [ 647.865458] Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache netfs v] > > > > [ 647.874688] ---[ end trace 0000000000000000 ]--- > > > > [ 647.876013] RIP: 0010:domain_unmap+0x48/0x110 > > > > [ 647.878306] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 > > > > [ 647.884581] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 > > > > [ 647.886308] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b > > > > [ 647.888775] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 > > > > [ 647.890295] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 > > > > [ 647.891660] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff > > > > [ 647.893019] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 > > > > [ 647.894506] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 > > > > [ 647.895963] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > > > > [ 647.897348] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 > > > > [ 647.898719] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > > > > > > > > Cc: stable@vger.kernel.org > > > > Fixes: 4c8cf31885f6 ("vhost: introduce vDPA-based backend") > > > > Signed-off-by: Cindy Lu <lulu@redhat.com> > > > > --- > > > > drivers/vhost/vdpa.c | 10 ++++++++-- > > > > 1 file changed, 8 insertions(+), 2 deletions(-) > > > > > > > > diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c > > > > index 166044642fd5..e5a07751bf45 100644 > > > > --- a/drivers/vhost/vdpa.c > > > > +++ b/drivers/vhost/vdpa.c > > > > @@ -692,6 +692,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > > > > struct vhost_iotlb_map *map; > > > > struct page *page; > > > > unsigned long pfn, pinned; > > > > + struct vdpa_device *vdpa = v->vdpa; > > > > + const struct vdpa_config_ops *ops = vdpa->config; > > > > > > > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > > > > pinned = PFN_DOWN(map->size); > > > > @@ -703,6 +705,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, > > > > unpin_user_page(page); > > > > } > > > > atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); > > > > + if ((ops->dma_map == NULL) && (ops->set_map == NULL)) > > > > + iommu_unmap(v->domain, map->start, map->size); > > > > > > I think we'd better move the ops->dma_unmap() here as well as iommu_unmap()? > > > > > > > vhost_iotlb_map_free(iotlb, map); > > > > } > > > > } > > > > @@ -713,11 +717,15 @@ static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, > > > > { > > > > struct vhost_iotlb_map *map; > > > > struct vdpa_map_file *map_file; > > > > + struct vdpa_device *vdpa = v->vdpa; > > > > + const struct vdpa_config_ops *ops = vdpa->config; > > > > > > > > while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { > > > > map_file = (struct vdpa_map_file *)map->opaque; > > > > fput(map_file->file); > > > > kfree(map_file); > > > > + if (ops->set_map == NULL) > > > > + iommu_unmap(v->domain, map->start, map->size); > > > > > > Need to check where we have dma_unmap() and call that if it exists? > > > > > > Thanks > > > > > Hi Jason， > > I think these functions are called in vhost_vdpa_unmap, > > Do you want to separate the function in vhost_vdpa_unmap > > and move it to vhost_vdpa_va_unmap and vhost_vdpa_pa_unmap? I > > I meant dma_map()/dma_unmap() should be functional equivalent to > iommu_map/unmap(). That means we should unmap exactly what is mapped > before (vDPA parent may call iommu_unmap in its own dma_unmap() if it > needs). If we move the iommu_unmap() from vhost_vdpa_unmap() to > vhost_vdpa_{pa|va}_umap, we should move dma_unmap() as well. > > Thanks > Got it.thanks Jason, I will change this part Thanks Cindy > > thanks > > cindy > > > > > > vhost_iotlb_map_free(iotlb, map); > > > > } > > > > } > > > > @@ -805,8 +813,6 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v, > > > > } else if (ops->set_map) { > > > > if (!v->in_batch) > > > > ops->set_map(vdpa, asid, iotlb); > > > > - } else { > > > > - iommu_unmap(v->domain, iova, size); > > > > } > > > > > > > > /* If we are in the middle of batch processing, delay the free > > > > -- > > > > 2.34.3 > > > > > > > > > >
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 166044642fd5..e5a07751bf45 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -692,6 +692,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb_map *map; struct page *page; unsigned long pfn, pinned; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { pinned = PFN_DOWN(map->size); @@ -703,6 +705,8 @@ static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, unpin_user_page(page); } atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm); + if ((ops->dma_map == NULL) && (ops->set_map == NULL)) + iommu_unmap(v->domain, map->start, map->size); vhost_iotlb_map_free(iotlb, map); } } @@ -713,11 +717,15 @@ static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, { struct vhost_iotlb_map *map; struct vdpa_map_file *map_file; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { map_file = (struct vdpa_map_file *)map->opaque; fput(map_file->file); kfree(map_file); + if (ops->set_map == NULL) + iommu_unmap(v->domain, map->start, map->size); vhost_iotlb_map_free(iotlb, map); } } @@ -805,8 +813,6 @@ static void vhost_vdpa_unmap(struct vhost_vdpa *v, } else if (ops->set_map) { if (!v->in_batch) ops->set_map(vdpa, asid, iotlb); - } else { - iommu_unmap(v->domain, iova, size); } /* If we are in the middle of batch processing, delay the free
While testing in vIOMMU, sometimes guest will unmap very large memory, which will cause the crash. To fix this,Move the iommu_unmap to vhost_vdpa_pa_unmap/vhost_vdpa_va_unmap and only unmap the memory that saved in iotlb. Call Trace: [ 647.820144] ------------[ cut here ]------------ [ 647.820848] kernel BUG at drivers/iommu/intel/iommu.c:1174! [ 647.821486] invalid opcode: 0000 [#1] PREEMPT SMP PTI [ 647.822082] CPU: 10 PID: 1181 Comm: qemu-system-x86 Not tainted 6.0.0-rc1home_lulu_2452_lulu7_vhost+ #62 [ 647.823139] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.15.0-29-g6a62e0cb0dfe-prebuilt.qem4 [ 647.824365] RIP: 0010:domain_unmap+0x48/0x110 [ 647.825424] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 [ 647.828064] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 [ 647.828973] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b [ 647.830083] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 [ 647.831214] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 [ 647.832388] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff [ 647.833668] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 [ 647.834782] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 [ 647.836004] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 647.836990] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 [ 647.838107] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 647.839283] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 647.840666] Call Trace: [ 647.841437] <TASK> [ 647.842107] intel_iommu_unmap_pages+0x93/0x140 [ 647.843112] __iommu_unmap+0x91/0x1b0 [ 647.844003] iommu_unmap+0x6a/0x95 [ 647.844885] vhost_vdpa_unmap+0x1de/0x1f0 [vhost_vdpa] [ 647.845985] vhost_vdpa_process_iotlb_msg+0xf0/0x90b [vhost_vdpa] [ 647.847235] ? _raw_spin_unlock+0x15/0x30 [ 647.848181] ? _copy_from_iter+0x8c/0x580 [ 647.849137] vhost_chr_write_iter+0xb3/0x430 [vhost] [ 647.850126] vfs_write+0x1e4/0x3a0 [ 647.850897] ksys_write+0x53/0xd0 [ 647.851688] do_syscall_64+0x3a/0x90 [ 647.852508] entry_SYSCALL_64_after_hwframe+0x63/0xcd [ 647.853457] RIP: 0033:0x7f7734ef9f4f [ 647.854408] Code: 89 54 24 18 48 89 74 24 10 89 7c 24 08 e8 29 76 f8 ff 48 8b 54 24 18 48 8b 74 24 10 41 89 c8 [ 647.857217] RSP: 002b:00007f772ec8f040 EFLAGS: 00000293 ORIG_RAX: 0000000000000001 [ 647.858486] RAX: ffffffffffffffda RBX: 00000000fef00000 RCX: 00007f7734ef9f4f [ 647.859713] RDX: 0000000000000048 RSI: 00007f772ec8f090 RDI: 0000000000000010 [ 647.860942] RBP: 00007f772ec8f1a0 R08: 0000000000000000 R09: 0000000000000000 [ 647.862206] R10: 0000000000000001 R11: 0000000000000293 R12: 0000000000000010 [ 647.863446] R13: 0000000000000002 R14: 0000000000000000 R15: ffffffff01100000 [ 647.864692] </TASK> [ 647.865458] Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache netfs v] [ 647.874688] ---[ end trace 0000000000000000 ]--- [ 647.876013] RIP: 0010:domain_unmap+0x48/0x110 [ 647.878306] Code: 48 89 fb 8d 4c f6 1e 39 c1 0f 4f c8 83 e9 0c 83 f9 3f 7f 18 48 89 e8 48 d3 e8 48 85 c0 75 59 [ 647.884581] RSP: 0018:ffffae5340c0bbf0 EFLAGS: 00010202 [ 647.886308] RAX: 0000000000000001 RBX: ffff921793d10540 RCX: 000000000000001b [ 647.888775] RDX: 00000000080000ff RSI: 0000000000000001 RDI: ffff921793d10540 [ 647.890295] RBP: 0000000007fc0100 R08: ffffae5340c0bcd0 R09: 0000000000000003 [ 647.891660] R10: 0000007fc0100000 R11: 0000000000100000 R12: 00000000080000ff [ 647.893019] R13: ffffae5340c0bcd0 R14: ffff921793d10590 R15: 0000008000100000 [ 647.894506] FS: 00007f772ec90640(0000) GS:ffff921ce7a80000(0000) knlGS:0000000000000000 [ 647.895963] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 647.897348] CR2: 00007f02c27a3a20 CR3: 0000000101b0c006 CR4: 0000000000372ee0 [ 647.898719] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 Cc: stable@vger.kernel.org Fixes: 4c8cf31885f6 ("vhost: introduce vDPA-based backend") Signed-off-by: Cindy Lu <lulu@redhat.com> --- drivers/vhost/vdpa.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)