@@ -40,10 +40,10 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj)
int n_sgt = bo->base.base.size / SZ_2M;
for (i = 0; i < n_sgt; i++) {
- if (bo->sgts[i].sgl) {
- dma_unmap_sgtable(pfdev->dev, &bo->sgts[i],
+ if (bo->sgts[i]) {
+ dma_unmap_sgtable(pfdev->dev, bo->sgts[i],
DMA_BIDIRECTIONAL, 0);
- sg_free_table(&bo->sgts[i]);
+ sg_free_table(bo->sgts[i]);
}
}
kvfree(bo->sgts);
@@ -274,7 +274,11 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
if (flags & PANFROST_BO_HEAP)
size = roundup(size, SZ_2M);
- shmem = drm_gem_shmem_create(dev, size);
+ if (flags & PANFROST_BO_HEAP)
+ shmem = drm_gem_shmem_create_sparse(dev, size);
+ else
+ shmem = drm_gem_shmem_create(dev, size);
+
if (IS_ERR(shmem))
return ERR_CAST(shmem);
@@ -11,7 +11,7 @@ struct panfrost_mmu;
struct panfrost_gem_object {
struct drm_gem_shmem_object base;
- struct sg_table *sgts;
+ struct sg_table **sgts;
/*
* Use a list for now. If searching a mapping ever becomes the
@@ -441,14 +441,11 @@ addr_to_mapping(struct panfrost_device *pfdev, int as, u64 addr)
static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
u64 addr)
{
- int ret, i;
struct panfrost_gem_mapping *bomapping;
struct panfrost_gem_object *bo;
- struct address_space *mapping;
- struct drm_gem_object *obj;
pgoff_t page_offset;
struct sg_table *sgt;
- struct page **pages;
+ int ret = 0;
bomapping = addr_to_mapping(pfdev, as, addr);
if (!bomapping)
@@ -459,94 +456,44 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as,
dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)",
bomapping->mmnode.start << PAGE_SHIFT);
ret = -EINVAL;
- goto err_bo;
+ goto fault_out;
}
WARN_ON(bomapping->mmu->as != as);
/* Assume 2MB alignment and size multiple */
addr &= ~((u64)SZ_2M - 1);
- page_offset = addr >> PAGE_SHIFT;
- page_offset -= bomapping->mmnode.start;
+ page_offset = (addr >> PAGE_SHIFT) - bomapping->mmnode.start;
- obj = &bo->base.base;
-
- dma_resv_lock(obj->resv, NULL);
-
- if (!bo->base.pages) {
+ if (!bo->sgts) {
bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M,
- sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO);
+ sizeof(struct sg_table *), GFP_KERNEL | __GFP_ZERO);
if (!bo->sgts) {
ret = -ENOMEM;
- goto err_unlock;
- }
-
- pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT,
- sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
- if (!pages) {
- kvfree(bo->sgts);
- bo->sgts = NULL;
- ret = -ENOMEM;
- goto err_unlock;
- }
- bo->base.pages = pages;
- bo->base.pages_use_count = 1;
- } else {
- pages = bo->base.pages;
- if (pages[page_offset]) {
- /* Pages are already mapped, bail out. */
- goto out;
+ goto fault_out;
}
}
- mapping = bo->base.base.filp->f_mapping;
- mapping_set_unevictable(mapping);
+ sgt = drm_gem_shmem_get_sparse_pages_sgt(&bo->base, NUM_FAULT_PAGES, page_offset);
+ if (IS_ERR(sgt)) {
+ if (WARN_ON(PTR_ERR(sgt) != -EEXIST))
+ ret = PTR_ERR(sgt);
+ else
+ ret = 0;
- for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) {
- /* Can happen if the last fault only partially filled this
- * section of the pages array before failing. In that case
- * we skip already filled pages.
- */
- if (pages[i])
- continue;
-
- pages[i] = shmem_read_mapping_page(mapping, i);
- if (IS_ERR(pages[i])) {
- ret = PTR_ERR(pages[i]);
- pages[i] = NULL;
- goto err_unlock;
- }
+ goto fault_out;
}
- sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)];
- ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
- NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
- if (ret)
- goto err_unlock;
-
- ret = dma_map_sgtable(pfdev->dev, sgt, DMA_BIDIRECTIONAL, 0);
- if (ret)
- goto err_map;
-
mmu_map_sg(pfdev, bomapping->mmu, addr,
IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt);
+ bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)] = sgt;
+
bomapping->active = true;
bo->heap_rss_size += SZ_2M;
dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr);
-out:
- dma_resv_unlock(obj->resv);
-
- panfrost_gem_mapping_put(bomapping);
-
- return 0;
-
-err_map:
- sg_free_table(sgt);
-err_unlock:
- dma_resv_unlock(obj->resv);
-err_bo:
+fault_out:
panfrost_gem_mapping_put(bomapping);
return ret;
}
Panfrost heap BOs grow on demand when the GPU triggers a page fault after accessing an address within the BO's virtual range. We still store the sgts we get back from the shmem sparse allocation function, since it was decided management of sparse memory SGTs should be done by client drivers rather than the shmem subsystem. Signed-off-by: Adrián Larumbe <adrian.larumbe@collabora.com> --- drivers/gpu/drm/panfrost/panfrost_gem.c | 12 ++-- drivers/gpu/drm/panfrost/panfrost_gem.h | 2 +- drivers/gpu/drm/panfrost/panfrost_mmu.c | 85 +++++-------------------- 3 files changed, 25 insertions(+), 74 deletions(-)