| Message ID | 20210512142648.666476-17-andrey.grodzovsky@amd.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | RFC Support hot device unplug in amdgpu | expand |
Ping Andrey On 2021-05-12 10:26 a.m., Andrey Grodzovsky wrote: > Access to those must be prevented post pci_remove > > v6: Drop BOs list, unampping VRAM BAR is enough. > > Signed-off-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com> > --- > drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 24 +++++++++++++++++++--- > drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + > drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ---- > 3 files changed, 22 insertions(+), 7 deletions(-) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > index f7cca25c0fa0..73cbc3c7453f 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > @@ -3666,6 +3666,25 @@ int amdgpu_device_init(struct amdgpu_device *adev, > return r; > } > > +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) > +{ > + /* Clear all CPU mappings pointing to this device */ > + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); > + > + /* Unmap all mapped bars - Doorbell, registers and VRAM */ > + amdgpu_device_doorbell_fini(adev); > + > + iounmap(adev->rmmio); > + adev->rmmio = NULL; > + if (adev->mman.aper_base_kaddr) > + iounmap(adev->mman.aper_base_kaddr); > + adev->mman.aper_base_kaddr = NULL; > + > + /* Memory manager related */ > + arch_phys_wc_del(adev->gmc.vram_mtrr); > + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); > +} > + > /** > * amdgpu_device_fini - tear down the driver > * > @@ -3712,6 +3731,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) > amdgpu_device_ip_fini_early(adev); > > amdgpu_gart_dummy_page_fini(adev); > + > + amdgpu_device_unmap_mmio(adev); > } > > void amdgpu_device_fini_sw(struct amdgpu_device *adev) > @@ -3739,9 +3760,6 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) > } > if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) > vga_client_register(adev->pdev, NULL, NULL, NULL); > - iounmap(adev->rmmio); > - adev->rmmio = NULL; > - amdgpu_device_doorbell_fini(adev); > > if (IS_ENABLED(CONFIG_PERF_EVENTS)) > amdgpu_pmu_fini(adev); > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > index 0adffcace326..882fb49f3c41 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > @@ -533,6 +533,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, > return -ENOMEM; > drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); > INIT_LIST_HEAD(&bo->shadow_list); > + > bo->vm_bo = NULL; > bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : > bp->domain; > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > index 0d54e70278ca..58ad2fecc9e3 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > @@ -1841,10 +1841,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) > amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); > amdgpu_ttm_fw_reserve_vram_fini(adev); > > - if (adev->mman.aper_base_kaddr) > - iounmap(adev->mman.aper_base_kaddr); > - adev->mman.aper_base_kaddr = NULL; > - > amdgpu_vram_mgr_fini(adev); > amdgpu_gtt_mgr_fini(adev); > ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); >
Ping Andrey On 2021-05-14 10:42 a.m., Andrey Grodzovsky wrote: > Ping > > Andrey > > On 2021-05-12 10:26 a.m., Andrey Grodzovsky wrote: >> Access to those must be prevented post pci_remove >> >> v6: Drop BOs list, unampping VRAM BAR is enough. >> >> Signed-off-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com> >> --- >> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 24 +++++++++++++++++++--- >> drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + >> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ---- >> 3 files changed, 22 insertions(+), 7 deletions(-) >> >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> index f7cca25c0fa0..73cbc3c7453f 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> @@ -3666,6 +3666,25 @@ int amdgpu_device_init(struct amdgpu_device *adev, >> return r; >> } >> +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) >> +{ >> + /* Clear all CPU mappings pointing to this device */ >> + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); >> + >> + /* Unmap all mapped bars - Doorbell, registers and VRAM */ >> + amdgpu_device_doorbell_fini(adev); >> + >> + iounmap(adev->rmmio); >> + adev->rmmio = NULL; >> + if (adev->mman.aper_base_kaddr) >> + iounmap(adev->mman.aper_base_kaddr); >> + adev->mman.aper_base_kaddr = NULL; >> + >> + /* Memory manager related */ >> + arch_phys_wc_del(adev->gmc.vram_mtrr); >> + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); >> +} >> + >> /** >> * amdgpu_device_fini - tear down the driver >> * >> @@ -3712,6 +3731,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device >> *adev) >> amdgpu_device_ip_fini_early(adev); >> amdgpu_gart_dummy_page_fini(adev); >> + >> + amdgpu_device_unmap_mmio(adev); >> } >> void amdgpu_device_fini_sw(struct amdgpu_device *adev) >> @@ -3739,9 +3760,6 @@ void amdgpu_device_fini_sw(struct amdgpu_device >> *adev) >> } >> if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) >> vga_client_register(adev->pdev, NULL, NULL, NULL); >> - iounmap(adev->rmmio); >> - adev->rmmio = NULL; >> - amdgpu_device_doorbell_fini(adev); >> if (IS_ENABLED(CONFIG_PERF_EVENTS)) >> amdgpu_pmu_fini(adev); >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >> b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >> index 0adffcace326..882fb49f3c41 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >> @@ -533,6 +533,7 @@ static int amdgpu_bo_do_create(struct >> amdgpu_device *adev, >> return -ENOMEM; >> drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, >> size); >> INIT_LIST_HEAD(&bo->shadow_list); >> + >> bo->vm_bo = NULL; >> bo->preferred_domains = bp->preferred_domain ? >> bp->preferred_domain : >> bp->domain; >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >> index 0d54e70278ca..58ad2fecc9e3 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >> @@ -1841,10 +1841,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) >> amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); >> amdgpu_ttm_fw_reserve_vram_fini(adev); >> - if (adev->mman.aper_base_kaddr) >> - iounmap(adev->mman.aper_base_kaddr); >> - adev->mman.aper_base_kaddr = NULL; >> - >> amdgpu_vram_mgr_fini(adev); >> amdgpu_gtt_mgr_fini(adev); >> ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); >>
On Wed, May 12, 2021 at 10:27 AM Andrey Grodzovsky <andrey.grodzovsky@amd.com> wrote: > > Access to those must be prevented post pci_remove > > v6: Drop BOs list, unampping VRAM BAR is enough. > > Signed-off-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com> > --- > drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 24 +++++++++++++++++++--- > drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + > drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ---- > 3 files changed, 22 insertions(+), 7 deletions(-) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > index f7cca25c0fa0..73cbc3c7453f 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > @@ -3666,6 +3666,25 @@ int amdgpu_device_init(struct amdgpu_device *adev, > return r; > } > > +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) > +{ > + /* Clear all CPU mappings pointing to this device */ > + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); > + > + /* Unmap all mapped bars - Doorbell, registers and VRAM */ > + amdgpu_device_doorbell_fini(adev); > + > + iounmap(adev->rmmio); > + adev->rmmio = NULL; > + if (adev->mman.aper_base_kaddr) > + iounmap(adev->mman.aper_base_kaddr); > + adev->mman.aper_base_kaddr = NULL; > + > + /* Memory manager related */ I think we need: if (!adev->gmc.xgmi.connected_to_cpu) { around these two to mirror amdgpu_bo_fini(). Alex > + arch_phys_wc_del(adev->gmc.vram_mtrr); > + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); > +} > + > /** > * amdgpu_device_fini - tear down the driver > * > @@ -3712,6 +3731,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) > amdgpu_device_ip_fini_early(adev); > > amdgpu_gart_dummy_page_fini(adev); > + > + amdgpu_device_unmap_mmio(adev); > } > > void amdgpu_device_fini_sw(struct amdgpu_device *adev) > @@ -3739,9 +3760,6 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) > } > if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) > vga_client_register(adev->pdev, NULL, NULL, NULL); > - iounmap(adev->rmmio); > - adev->rmmio = NULL; > - amdgpu_device_doorbell_fini(adev); > > if (IS_ENABLED(CONFIG_PERF_EVENTS)) > amdgpu_pmu_fini(adev); > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > index 0adffcace326..882fb49f3c41 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > @@ -533,6 +533,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, > return -ENOMEM; > drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); > INIT_LIST_HEAD(&bo->shadow_list); > + > bo->vm_bo = NULL; > bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : > bp->domain; > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > index 0d54e70278ca..58ad2fecc9e3 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > @@ -1841,10 +1841,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) > amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); > amdgpu_ttm_fw_reserve_vram_fini(adev); > > - if (adev->mman.aper_base_kaddr) > - iounmap(adev->mman.aper_base_kaddr); > - adev->mman.aper_base_kaddr = NULL; > - > amdgpu_vram_mgr_fini(adev); > amdgpu_gtt_mgr_fini(adev); > ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); > -- > 2.25.1 >
On 2021-05-17 1:43 p.m., Alex Deucher wrote: > On Wed, May 12, 2021 at 10:27 AM Andrey Grodzovsky > <andrey.grodzovsky@amd.com> wrote: >> >> Access to those must be prevented post pci_remove >> >> v6: Drop BOs list, unampping VRAM BAR is enough. >> >> Signed-off-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com> >> --- >> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 24 +++++++++++++++++++--- >> drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + >> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ---- >> 3 files changed, 22 insertions(+), 7 deletions(-) >> >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> index f7cca25c0fa0..73cbc3c7453f 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> @@ -3666,6 +3666,25 @@ int amdgpu_device_init(struct amdgpu_device *adev, >> return r; >> } >> >> +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) >> +{ >> + /* Clear all CPU mappings pointing to this device */ >> + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); >> + >> + /* Unmap all mapped bars - Doorbell, registers and VRAM */ >> + amdgpu_device_doorbell_fini(adev); >> + >> + iounmap(adev->rmmio); >> + adev->rmmio = NULL; >> + if (adev->mman.aper_base_kaddr) >> + iounmap(adev->mman.aper_base_kaddr); >> + adev->mman.aper_base_kaddr = NULL; >> + >> + /* Memory manager related */ > > I think we need: > if (!adev->gmc.xgmi.connected_to_cpu) { > around these two to mirror amdgpu_bo_fini(). > > Alex I am working of off drm-misc-next and here amdgpu_xgmi doesn't have connected_to_cpu yet. Andrey > >> + arch_phys_wc_del(adev->gmc.vram_mtrr); >> + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); >> +} >> + >> /** >> * amdgpu_device_fini - tear down the driver >> * >> @@ -3712,6 +3731,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) >> amdgpu_device_ip_fini_early(adev); >> >> amdgpu_gart_dummy_page_fini(adev); >> + >> + amdgpu_device_unmap_mmio(adev); >> } >> >> void amdgpu_device_fini_sw(struct amdgpu_device *adev) >> @@ -3739,9 +3760,6 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) >> } >> if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) >> vga_client_register(adev->pdev, NULL, NULL, NULL); >> - iounmap(adev->rmmio); >> - adev->rmmio = NULL; >> - amdgpu_device_doorbell_fini(adev); >> >> if (IS_ENABLED(CONFIG_PERF_EVENTS)) >> amdgpu_pmu_fini(adev); >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >> index 0adffcace326..882fb49f3c41 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >> @@ -533,6 +533,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, >> return -ENOMEM; >> drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); >> INIT_LIST_HEAD(&bo->shadow_list); >> + >> bo->vm_bo = NULL; >> bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : >> bp->domain; >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >> index 0d54e70278ca..58ad2fecc9e3 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >> @@ -1841,10 +1841,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) >> amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); >> amdgpu_ttm_fw_reserve_vram_fini(adev); >> >> - if (adev->mman.aper_base_kaddr) >> - iounmap(adev->mman.aper_base_kaddr); >> - adev->mman.aper_base_kaddr = NULL; >> - >> amdgpu_vram_mgr_fini(adev); >> amdgpu_gtt_mgr_fini(adev); >> ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); >> -- >> 2.25.1 >>
On Mon, May 17, 2021 at 2:46 PM Andrey Grodzovsky <andrey.grodzovsky@amd.com> wrote: > > > > On 2021-05-17 1:43 p.m., Alex Deucher wrote: > > On Wed, May 12, 2021 at 10:27 AM Andrey Grodzovsky > > <andrey.grodzovsky@amd.com> wrote: > >> > >> Access to those must be prevented post pci_remove > >> > >> v6: Drop BOs list, unampping VRAM BAR is enough. > >> > >> Signed-off-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com> > >> --- > >> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 24 +++++++++++++++++++--- > >> drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + > >> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ---- > >> 3 files changed, 22 insertions(+), 7 deletions(-) > >> > >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > >> index f7cca25c0fa0..73cbc3c7453f 100644 > >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > >> @@ -3666,6 +3666,25 @@ int amdgpu_device_init(struct amdgpu_device *adev, > >> return r; > >> } > >> > >> +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) > >> +{ > >> + /* Clear all CPU mappings pointing to this device */ > >> + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); > >> + > >> + /* Unmap all mapped bars - Doorbell, registers and VRAM */ > >> + amdgpu_device_doorbell_fini(adev); > >> + > >> + iounmap(adev->rmmio); > >> + adev->rmmio = NULL; > >> + if (adev->mman.aper_base_kaddr) > >> + iounmap(adev->mman.aper_base_kaddr); > >> + adev->mman.aper_base_kaddr = NULL; > >> + > >> + /* Memory manager related */ > > > > I think we need: > > if (!adev->gmc.xgmi.connected_to_cpu) { > > around these two to mirror amdgpu_bo_fini(). > > > > Alex > > I am working of off drm-misc-next and here amdgpu_xgmi > doesn't have connected_to_cpu yet. Ah, right. Ok. Do we need to remove the code from bo_fini() if we handle it here now? Alex > > Andrey > > > > >> + arch_phys_wc_del(adev->gmc.vram_mtrr); > >> + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); > >> +} > >> + > >> /** > >> * amdgpu_device_fini - tear down the driver > >> * > >> @@ -3712,6 +3731,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) > >> amdgpu_device_ip_fini_early(adev); > >> > >> amdgpu_gart_dummy_page_fini(adev); > >> + > >> + amdgpu_device_unmap_mmio(adev); > >> } > >> > >> void amdgpu_device_fini_sw(struct amdgpu_device *adev) > >> @@ -3739,9 +3760,6 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) > >> } > >> if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) > >> vga_client_register(adev->pdev, NULL, NULL, NULL); > >> - iounmap(adev->rmmio); > >> - adev->rmmio = NULL; > >> - amdgpu_device_doorbell_fini(adev); > >> > >> if (IS_ENABLED(CONFIG_PERF_EVENTS)) > >> amdgpu_pmu_fini(adev); > >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > >> index 0adffcace326..882fb49f3c41 100644 > >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c > >> @@ -533,6 +533,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, > >> return -ENOMEM; > >> drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); > >> INIT_LIST_HEAD(&bo->shadow_list); > >> + > >> bo->vm_bo = NULL; > >> bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : > >> bp->domain; > >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > >> index 0d54e70278ca..58ad2fecc9e3 100644 > >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c > >> @@ -1841,10 +1841,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) > >> amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); > >> amdgpu_ttm_fw_reserve_vram_fini(adev); > >> > >> - if (adev->mman.aper_base_kaddr) > >> - iounmap(adev->mman.aper_base_kaddr); > >> - adev->mman.aper_base_kaddr = NULL; > >> - > >> amdgpu_vram_mgr_fini(adev); > >> amdgpu_gtt_mgr_fini(adev); > >> ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); > >> -- > >> 2.25.1 > >>
On 2021-05-17 2:56 p.m., Alex Deucher wrote: > On Mon, May 17, 2021 at 2:46 PM Andrey Grodzovsky > <andrey.grodzovsky@amd.com> wrote: >> >> >> >> On 2021-05-17 1:43 p.m., Alex Deucher wrote: >>> On Wed, May 12, 2021 at 10:27 AM Andrey Grodzovsky >>> <andrey.grodzovsky@amd.com> wrote: >>>> >>>> Access to those must be prevented post pci_remove >>>> >>>> v6: Drop BOs list, unampping VRAM BAR is enough. >>>> >>>> Signed-off-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com> >>>> --- >>>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 24 +++++++++++++++++++--- >>>> drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + >>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ---- >>>> 3 files changed, 22 insertions(+), 7 deletions(-) >>>> >>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>>> index f7cca25c0fa0..73cbc3c7453f 100644 >>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>>> @@ -3666,6 +3666,25 @@ int amdgpu_device_init(struct amdgpu_device *adev, >>>> return r; >>>> } >>>> >>>> +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) >>>> +{ >>>> + /* Clear all CPU mappings pointing to this device */ >>>> + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); >>>> + >>>> + /* Unmap all mapped bars - Doorbell, registers and VRAM */ >>>> + amdgpu_device_doorbell_fini(adev); >>>> + >>>> + iounmap(adev->rmmio); >>>> + adev->rmmio = NULL; >>>> + if (adev->mman.aper_base_kaddr) >>>> + iounmap(adev->mman.aper_base_kaddr); >>>> + adev->mman.aper_base_kaddr = NULL; >>>> + >>>> + /* Memory manager related */ >>> >>> I think we need: >>> if (!adev->gmc.xgmi.connected_to_cpu) { >>> around these two to mirror amdgpu_bo_fini(). >>> >>> Alex >> >> I am working of off drm-misc-next and here amdgpu_xgmi >> doesn't have connected_to_cpu yet. > > Ah, right. Ok. Do we need to remove the code from bo_fini() if we > handle it here now? > > Alex My bad, I was on older kernel due to fixing internal ticket last week, in latest drm-misc-next there is connected_to_cpu and so I fixed everything as you asked. Will resend in a moment. Andrey > > >> >> Andrey >> >>> >>>> + arch_phys_wc_del(adev->gmc.vram_mtrr); >>>> + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); >>>> +} >>>> + >>>> /** >>>> * amdgpu_device_fini - tear down the driver >>>> * >>>> @@ -3712,6 +3731,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) >>>> amdgpu_device_ip_fini_early(adev); >>>> >>>> amdgpu_gart_dummy_page_fini(adev); >>>> + >>>> + amdgpu_device_unmap_mmio(adev); >>>> } >>>> >>>> void amdgpu_device_fini_sw(struct amdgpu_device *adev) >>>> @@ -3739,9 +3760,6 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) >>>> } >>>> if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) >>>> vga_client_register(adev->pdev, NULL, NULL, NULL); >>>> - iounmap(adev->rmmio); >>>> - adev->rmmio = NULL; >>>> - amdgpu_device_doorbell_fini(adev); >>>> >>>> if (IS_ENABLED(CONFIG_PERF_EVENTS)) >>>> amdgpu_pmu_fini(adev); >>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >>>> index 0adffcace326..882fb49f3c41 100644 >>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c >>>> @@ -533,6 +533,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, >>>> return -ENOMEM; >>>> drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); >>>> INIT_LIST_HEAD(&bo->shadow_list); >>>> + >>>> bo->vm_bo = NULL; >>>> bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : >>>> bp->domain; >>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >>>> index 0d54e70278ca..58ad2fecc9e3 100644 >>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c >>>> @@ -1841,10 +1841,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) >>>> amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); >>>> amdgpu_ttm_fw_reserve_vram_fini(adev); >>>> >>>> - if (adev->mman.aper_base_kaddr) >>>> - iounmap(adev->mman.aper_base_kaddr); >>>> - adev->mman.aper_base_kaddr = NULL; >>>> - >>>> amdgpu_vram_mgr_fini(adev); >>>> amdgpu_gtt_mgr_fini(adev); >>>> ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); >>>> -- >>>> 2.25.1 >>>>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index f7cca25c0fa0..73cbc3c7453f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3666,6 +3666,25 @@ int amdgpu_device_init(struct amdgpu_device *adev, return r; } +static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) +{ + /* Clear all CPU mappings pointing to this device */ + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); + + /* Unmap all mapped bars - Doorbell, registers and VRAM */ + amdgpu_device_doorbell_fini(adev); + + iounmap(adev->rmmio); + adev->rmmio = NULL; + if (adev->mman.aper_base_kaddr) + iounmap(adev->mman.aper_base_kaddr); + adev->mman.aper_base_kaddr = NULL; + + /* Memory manager related */ + arch_phys_wc_del(adev->gmc.vram_mtrr); + arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); +} + /** * amdgpu_device_fini - tear down the driver * @@ -3712,6 +3731,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_device_ip_fini_early(adev); amdgpu_gart_dummy_page_fini(adev); + + amdgpu_device_unmap_mmio(adev); } void amdgpu_device_fini_sw(struct amdgpu_device *adev) @@ -3739,9 +3760,6 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) } if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) vga_client_register(adev->pdev, NULL, NULL, NULL); - iounmap(adev->rmmio); - adev->rmmio = NULL; - amdgpu_device_doorbell_fini(adev); if (IS_ENABLED(CONFIG_PERF_EVENTS)) amdgpu_pmu_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 0adffcace326..882fb49f3c41 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -533,6 +533,7 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, return -ENOMEM; drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); INIT_LIST_HEAD(&bo->shadow_list); + bo->vm_bo = NULL; bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : bp->domain; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 0d54e70278ca..58ad2fecc9e3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1841,10 +1841,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); amdgpu_ttm_fw_reserve_vram_fini(adev); - if (adev->mman.aper_base_kaddr) - iounmap(adev->mman.aper_base_kaddr); - adev->mman.aper_base_kaddr = NULL; - amdgpu_vram_mgr_fini(adev); amdgpu_gtt_mgr_fini(adev); ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS);
Access to those must be prevented post pci_remove v6: Drop BOs list, unampping VRAM BAR is enough. Signed-off-by: Andrey Grodzovsky <andrey.grodzovsky@amd.com> --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 24 +++++++++++++++++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ---- 3 files changed, 22 insertions(+), 7 deletions(-)