| Message ID | 20201007164426.1812530-2-daniel.vetter@ffwll.ch (mailing list archive) |
|---|---|
| State | Not Applicable |
| Headers | show |
| Series | follow_pfn and other iomap races | expand |
On 10/7/20 9:44 AM, Daniel Vetter wrote: > All we need are a pages array, pin_user_pages_fast can give us that > directly. Plus this avoids the entire raw pfn side of get_vaddr_frames. > > Signed-off-by: Daniel Vetter <daniel.vetter@intel.com> > Cc: Jason Gunthorpe <jgg@ziepe.ca> > Cc: Inki Dae <inki.dae@samsung.com> > Cc: Joonyoung Shim <jy0922.shim@samsung.com> > Cc: Seung-Woo Kim <sw0312.kim@samsung.com> > Cc: Kyungmin Park <kyungmin.park@samsung.com> > Cc: Kukjin Kim <kgene@kernel.org> > Cc: Krzysztof Kozlowski <krzk@kernel.org> > Cc: Andrew Morton <akpm@linux-foundation.org> > Cc: John Hubbard <jhubbard@nvidia.com> > Cc: Jérôme Glisse <jglisse@redhat.com> > Cc: Jan Kara <jack@suse.cz> > Cc: Dan Williams <dan.j.williams@intel.com> > Cc: linux-mm@kvack.org > Cc: linux-arm-kernel@lists.infradead.org > Cc: linux-samsung-soc@vger.kernel.org > Cc: linux-media@vger.kernel.org > --- > drivers/gpu/drm/exynos/Kconfig | 1 - > drivers/gpu/drm/exynos/exynos_drm_g2d.c | 48 ++++++++++++------------- > 2 files changed, 22 insertions(+), 27 deletions(-) > > diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig > index 6417f374b923..43257ef3c09d 100644 > --- a/drivers/gpu/drm/exynos/Kconfig > +++ b/drivers/gpu/drm/exynos/Kconfig > @@ -88,7 +88,6 @@ comment "Sub-drivers" > config DRM_EXYNOS_G2D > bool "G2D" > depends on VIDEO_SAMSUNG_S5P_G2D=n || COMPILE_TEST > - select FRAME_VECTOR > help > Choose this option if you want to use Exynos G2D for DRM. > > diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c > index 967a5cdc120e..c83f6faac9de 100644 > --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c > +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c > @@ -205,7 +205,8 @@ struct g2d_cmdlist_userptr { > dma_addr_t dma_addr; > unsigned long userptr; > unsigned long size; > - struct frame_vector *vec; > + struct page **pages; > + unsigned int npages; > struct sg_table *sgt; > atomic_t refcount; > bool in_pool; > @@ -378,7 +379,7 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, > bool force) > { > struct g2d_cmdlist_userptr *g2d_userptr = obj; > - struct page **pages; > + int i; The above line can also be deleted, see below. > > if (!obj) > return; > @@ -398,15 +399,11 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, > dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt, > DMA_BIDIRECTIONAL, 0); > > - pages = frame_vector_pages(g2d_userptr->vec); > - if (!IS_ERR(pages)) { > - int i; > + for (i = 0; i < g2d_userptr->npages; i++) > + set_page_dirty_lock(g2d_userptr->pages[i]); > > - for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) > - set_page_dirty_lock(pages[i]); > - } > - put_vaddr_frames(g2d_userptr->vec); > - frame_vector_destroy(g2d_userptr->vec); > + unpin_user_pages(g2d_userptr->pages, g2d_userptr->npages); > + kvfree(g2d_userptr->pages); You can avoid writing your own loop, and just simplify the whole thing down to two lines: unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages, true); kvfree(g2d_userptr->pages); > > if (!g2d_userptr->out_of_list) > list_del_init(&g2d_userptr->list); > @@ -474,35 +471,34 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, > offset = userptr & ~PAGE_MASK; > end = PAGE_ALIGN(userptr + size); > npages = (end - start) >> PAGE_SHIFT; > - g2d_userptr->vec = frame_vector_create(npages); > - if (!g2d_userptr->vec) { > + g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages), > + GFP_KERNEL); > + if (!g2d_userptr->pages) { > ret = -ENOMEM; > goto err_free; > } > > - ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, > - g2d_userptr->vec); > + ret = pin_user_pages_fast(start, npages, FOLL_FORCE | FOLL_WRITE, > + g2d_userptr->pages); > if (ret != npages) { > DRM_DEV_ERROR(g2d->dev, > "failed to get user pages from userptr.\n"); > if (ret < 0) > - goto err_destroy_framevec; > - ret = -EFAULT; > - goto err_put_framevec; > - } > - if (frame_vector_to_pages(g2d_userptr->vec) < 0) { > + goto err_destroy_pages; > + npages = ret; > ret = -EFAULT; > - goto err_put_framevec; > + goto err_unpin_pages; > } > + g2d_userptr->npages = npages; > > sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); > if (!sgt) { > ret = -ENOMEM; > - goto err_put_framevec; > + goto err_unpin_pages; > } > > ret = sg_alloc_table_from_pages(sgt, > - frame_vector_pages(g2d_userptr->vec), > + g2d_userptr->pages, > npages, offset, size, GFP_KERNEL); > if (ret < 0) { > DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n"); > @@ -538,11 +534,11 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, > err_free_sgt: > kfree(sgt); > > -err_put_framevec: > - put_vaddr_frames(g2d_userptr->vec); > +err_unpin_pages: > + unpin_user_pages(g2d_userptr->pages, npages); > > -err_destroy_framevec: > - frame_vector_destroy(g2d_userptr->vec); > +err_destroy_pages: > + kvfree(g2d_userptr->pages); > > err_free: > kfree(g2d_userptr); > The rest all looks good, you've avoided the usual API pitfalls. :) thanks,
On Wed, Oct 7, 2020 at 10:33 PM John Hubbard <jhubbard@nvidia.com> wrote: > > On 10/7/20 9:44 AM, Daniel Vetter wrote: > > All we need are a pages array, pin_user_pages_fast can give us that > > directly. Plus this avoids the entire raw pfn side of get_vaddr_frames. > > > > Signed-off-by: Daniel Vetter <daniel.vetter@intel.com> > > Cc: Jason Gunthorpe <jgg@ziepe.ca> > > Cc: Inki Dae <inki.dae@samsung.com> > > Cc: Joonyoung Shim <jy0922.shim@samsung.com> > > Cc: Seung-Woo Kim <sw0312.kim@samsung.com> > > Cc: Kyungmin Park <kyungmin.park@samsung.com> > > Cc: Kukjin Kim <kgene@kernel.org> > > Cc: Krzysztof Kozlowski <krzk@kernel.org> > > Cc: Andrew Morton <akpm@linux-foundation.org> > > Cc: John Hubbard <jhubbard@nvidia.com> > > Cc: Jérôme Glisse <jglisse@redhat.com> > > Cc: Jan Kara <jack@suse.cz> > > Cc: Dan Williams <dan.j.williams@intel.com> > > Cc: linux-mm@kvack.org > > Cc: linux-arm-kernel@lists.infradead.org > > Cc: linux-samsung-soc@vger.kernel.org > > Cc: linux-media@vger.kernel.org > > --- > > drivers/gpu/drm/exynos/Kconfig | 1 - > > drivers/gpu/drm/exynos/exynos_drm_g2d.c | 48 ++++++++++++------------- > > 2 files changed, 22 insertions(+), 27 deletions(-) > > > > diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig > > index 6417f374b923..43257ef3c09d 100644 > > --- a/drivers/gpu/drm/exynos/Kconfig > > +++ b/drivers/gpu/drm/exynos/Kconfig > > @@ -88,7 +88,6 @@ comment "Sub-drivers" > > config DRM_EXYNOS_G2D > > bool "G2D" > > depends on VIDEO_SAMSUNG_S5P_G2D=n || COMPILE_TEST > > - select FRAME_VECTOR > > help > > Choose this option if you want to use Exynos G2D for DRM. > > > > diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c > > index 967a5cdc120e..c83f6faac9de 100644 > > --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c > > +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c > > @@ -205,7 +205,8 @@ struct g2d_cmdlist_userptr { > > dma_addr_t dma_addr; > > unsigned long userptr; > > unsigned long size; > > - struct frame_vector *vec; > > + struct page **pages; > > + unsigned int npages; > > struct sg_table *sgt; > > atomic_t refcount; > > bool in_pool; > > @@ -378,7 +379,7 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, > > bool force) > > { > > struct g2d_cmdlist_userptr *g2d_userptr = obj; > > - struct page **pages; > > + int i; > > The above line can also be deleted, see below. > > > > > if (!obj) > > return; > > @@ -398,15 +399,11 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, > > dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt, > > DMA_BIDIRECTIONAL, 0); > > > > - pages = frame_vector_pages(g2d_userptr->vec); > > - if (!IS_ERR(pages)) { > > - int i; > > + for (i = 0; i < g2d_userptr->npages; i++) > > + set_page_dirty_lock(g2d_userptr->pages[i]); > > > > - for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) > > - set_page_dirty_lock(pages[i]); > > - } > > - put_vaddr_frames(g2d_userptr->vec); > > - frame_vector_destroy(g2d_userptr->vec); > > + unpin_user_pages(g2d_userptr->pages, g2d_userptr->npages); > > + kvfree(g2d_userptr->pages); > > You can avoid writing your own loop, and just simplify the whole thing down to > two lines: > > unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages, > true); > kvfree(g2d_userptr->pages); Oh nice, this is neat. I'll also roll it out in the habanalabs patch, that has the same thing. Well almost, it only uses set_page_dirty, not the _lock variant. But I have no idea whether that matters or not? -Daniel > > > > > > if (!g2d_userptr->out_of_list) > > list_del_init(&g2d_userptr->list); > > @@ -474,35 +471,34 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, > > offset = userptr & ~PAGE_MASK; > > end = PAGE_ALIGN(userptr + size); > > npages = (end - start) >> PAGE_SHIFT; > > - g2d_userptr->vec = frame_vector_create(npages); > > - if (!g2d_userptr->vec) { > > + g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages), > > + GFP_KERNEL); > > + if (!g2d_userptr->pages) { > > ret = -ENOMEM; > > goto err_free; > > } > > > > - ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, > > - g2d_userptr->vec); > > + ret = pin_user_pages_fast(start, npages, FOLL_FORCE | FOLL_WRITE, > > + g2d_userptr->pages); > > if (ret != npages) { > > DRM_DEV_ERROR(g2d->dev, > > "failed to get user pages from userptr.\n"); > > if (ret < 0) > > - goto err_destroy_framevec; > > - ret = -EFAULT; > > - goto err_put_framevec; > > - } > > - if (frame_vector_to_pages(g2d_userptr->vec) < 0) { > > + goto err_destroy_pages; > > + npages = ret; > > ret = -EFAULT; > > - goto err_put_framevec; > > + goto err_unpin_pages; > > } > > + g2d_userptr->npages = npages; > > > > sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); > > if (!sgt) { > > ret = -ENOMEM; > > - goto err_put_framevec; > > + goto err_unpin_pages; > > } > > > > ret = sg_alloc_table_from_pages(sgt, > > - frame_vector_pages(g2d_userptr->vec), > > + g2d_userptr->pages, > > npages, offset, size, GFP_KERNEL); > > if (ret < 0) { > > DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n"); > > @@ -538,11 +534,11 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, > > err_free_sgt: > > kfree(sgt); > > > > -err_put_framevec: > > - put_vaddr_frames(g2d_userptr->vec); > > +err_unpin_pages: > > + unpin_user_pages(g2d_userptr->pages, npages); > > > > -err_destroy_framevec: > > - frame_vector_destroy(g2d_userptr->vec); > > +err_destroy_pages: > > + kvfree(g2d_userptr->pages); > > > > err_free: > > kfree(g2d_userptr); > > > > The rest all looks good, you've avoided the usual API pitfalls. :) > > thanks, > -- > John Hubbard > NVIDIA
On 10/7/20 2:32 PM, Daniel Vetter wrote: > On Wed, Oct 7, 2020 at 10:33 PM John Hubbard <jhubbard@nvidia.com> wrote: >> >> On 10/7/20 9:44 AM, Daniel Vetter wrote: ... >>> @@ -398,15 +399,11 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, >>> dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt, >>> DMA_BIDIRECTIONAL, 0); >>> >>> - pages = frame_vector_pages(g2d_userptr->vec); >>> - if (!IS_ERR(pages)) { >>> - int i; >>> + for (i = 0; i < g2d_userptr->npages; i++) >>> + set_page_dirty_lock(g2d_userptr->pages[i]); >>> >>> - for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) >>> - set_page_dirty_lock(pages[i]); >>> - } >>> - put_vaddr_frames(g2d_userptr->vec); >>> - frame_vector_destroy(g2d_userptr->vec); >>> + unpin_user_pages(g2d_userptr->pages, g2d_userptr->npages); >>> + kvfree(g2d_userptr->pages); >> >> You can avoid writing your own loop, and just simplify the whole thing down to >> two lines: >> >> unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages, >> true); >> kvfree(g2d_userptr->pages); > > Oh nice, this is neat. I'll also roll it out in the habanalabs patch, > that has the same thing. Well almost, it only uses set_page_dirty, not > the _lock variant. But I have no idea whether that matters or not? It matters. And invariably, call sites that use set_page_dirty() instead of set_page_dirty_lock() were already wrong. Which is why I never had to provide anything like "unpin_user_pages_dirty (not locked)". Although in habanalabs case, I just reviewed patch 3 and I think they *were* correctly using set_page_dirty_lock()... thanks,
On Wed, Oct 7, 2020 at 11:37 PM John Hubbard <jhubbard@nvidia.com> wrote: > > On 10/7/20 2:32 PM, Daniel Vetter wrote: > > On Wed, Oct 7, 2020 at 10:33 PM John Hubbard <jhubbard@nvidia.com> wrote: > >> > >> On 10/7/20 9:44 AM, Daniel Vetter wrote: > ... > >>> @@ -398,15 +399,11 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, > >>> dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt, > >>> DMA_BIDIRECTIONAL, 0); > >>> > >>> - pages = frame_vector_pages(g2d_userptr->vec); > >>> - if (!IS_ERR(pages)) { > >>> - int i; > >>> + for (i = 0; i < g2d_userptr->npages; i++) > >>> + set_page_dirty_lock(g2d_userptr->pages[i]); > >>> > >>> - for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) > >>> - set_page_dirty_lock(pages[i]); > >>> - } > >>> - put_vaddr_frames(g2d_userptr->vec); > >>> - frame_vector_destroy(g2d_userptr->vec); > >>> + unpin_user_pages(g2d_userptr->pages, g2d_userptr->npages); > >>> + kvfree(g2d_userptr->pages); > >> > >> You can avoid writing your own loop, and just simplify the whole thing down to > >> two lines: > >> > >> unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages, > >> true); > >> kvfree(g2d_userptr->pages); > > > > Oh nice, this is neat. I'll also roll it out in the habanalabs patch, > > that has the same thing. Well almost, it only uses set_page_dirty, not > > the _lock variant. But I have no idea whether that matters or not? > > > It matters. And invariably, call sites that use set_page_dirty() instead > of set_page_dirty_lock() were already wrong. Which is why I never had to > provide anything like "unpin_user_pages_dirty (not locked)". > > Although in habanalabs case, I just reviewed patch 3 and I think they *were* > correctly using set_page_dirty_lock()... Yeah I mixed that up with some other code I read, habanalabs is using _lock. I have seen a pile of gup/pup code though that only uses set_page_dirty. And looking around I did not really parse the comment above set_page_dirty(). I guess just using the _lock variant shouldn't hurt too much. I've found a comment though from the infiniband umem notifier that it's sometimes called with the page locked, and sometimes not, so life is complicated there. But how it avoids races I didn't understand. -Daniel -- Daniel Vetter Software Engineer, Intel Corporation http://blog.ffwll.ch
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig index 6417f374b923..43257ef3c09d 100644 --- a/drivers/gpu/drm/exynos/Kconfig +++ b/drivers/gpu/drm/exynos/Kconfig @@ -88,7 +88,6 @@ comment "Sub-drivers" config DRM_EXYNOS_G2D bool "G2D" depends on VIDEO_SAMSUNG_S5P_G2D=n || COMPILE_TEST - select FRAME_VECTOR help Choose this option if you want to use Exynos G2D for DRM. diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c index 967a5cdc120e..c83f6faac9de 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c @@ -205,7 +205,8 @@ struct g2d_cmdlist_userptr { dma_addr_t dma_addr; unsigned long userptr; unsigned long size; - struct frame_vector *vec; + struct page **pages; + unsigned int npages; struct sg_table *sgt; atomic_t refcount; bool in_pool; @@ -378,7 +379,7 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, bool force) { struct g2d_cmdlist_userptr *g2d_userptr = obj; - struct page **pages; + int i; if (!obj) return; @@ -398,15 +399,11 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt, DMA_BIDIRECTIONAL, 0); - pages = frame_vector_pages(g2d_userptr->vec); - if (!IS_ERR(pages)) { - int i; + for (i = 0; i < g2d_userptr->npages; i++) + set_page_dirty_lock(g2d_userptr->pages[i]); - for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) - set_page_dirty_lock(pages[i]); - } - put_vaddr_frames(g2d_userptr->vec); - frame_vector_destroy(g2d_userptr->vec); + unpin_user_pages(g2d_userptr->pages, g2d_userptr->npages); + kvfree(g2d_userptr->pages); if (!g2d_userptr->out_of_list) list_del_init(&g2d_userptr->list); @@ -474,35 +471,34 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, offset = userptr & ~PAGE_MASK; end = PAGE_ALIGN(userptr + size); npages = (end - start) >> PAGE_SHIFT; - g2d_userptr->vec = frame_vector_create(npages); - if (!g2d_userptr->vec) { + g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages), + GFP_KERNEL); + if (!g2d_userptr->pages) { ret = -ENOMEM; goto err_free; } - ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, - g2d_userptr->vec); + ret = pin_user_pages_fast(start, npages, FOLL_FORCE | FOLL_WRITE, + g2d_userptr->pages); if (ret != npages) { DRM_DEV_ERROR(g2d->dev, "failed to get user pages from userptr.\n"); if (ret < 0) - goto err_destroy_framevec; - ret = -EFAULT; - goto err_put_framevec; - } - if (frame_vector_to_pages(g2d_userptr->vec) < 0) { + goto err_destroy_pages; + npages = ret; ret = -EFAULT; - goto err_put_framevec; + goto err_unpin_pages; } + g2d_userptr->npages = npages; sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); if (!sgt) { ret = -ENOMEM; - goto err_put_framevec; + goto err_unpin_pages; } ret = sg_alloc_table_from_pages(sgt, - frame_vector_pages(g2d_userptr->vec), + g2d_userptr->pages, npages, offset, size, GFP_KERNEL); if (ret < 0) { DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n"); @@ -538,11 +534,11 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, err_free_sgt: kfree(sgt); -err_put_framevec: - put_vaddr_frames(g2d_userptr->vec); +err_unpin_pages: + unpin_user_pages(g2d_userptr->pages, npages); -err_destroy_framevec: - frame_vector_destroy(g2d_userptr->vec); +err_destroy_pages: + kvfree(g2d_userptr->pages); err_free: kfree(g2d_userptr);
All we need are a pages array, pin_user_pages_fast can give us that directly. Plus this avoids the entire raw pfn side of get_vaddr_frames. Signed-off-by: Daniel Vetter <daniel.vetter@intel.com> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: Inki Dae <inki.dae@samsung.com> Cc: Joonyoung Shim <jy0922.shim@samsung.com> Cc: Seung-Woo Kim <sw0312.kim@samsung.com> Cc: Kyungmin Park <kyungmin.park@samsung.com> Cc: Kukjin Kim <kgene@kernel.org> Cc: Krzysztof Kozlowski <krzk@kernel.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: John Hubbard <jhubbard@nvidia.com> Cc: Jérôme Glisse <jglisse@redhat.com> Cc: Jan Kara <jack@suse.cz> Cc: Dan Williams <dan.j.williams@intel.com> Cc: linux-mm@kvack.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-samsung-soc@vger.kernel.org Cc: linux-media@vger.kernel.org --- drivers/gpu/drm/exynos/Kconfig | 1 - drivers/gpu/drm/exynos/exynos_drm_g2d.c | 48 ++++++++++++------------- 2 files changed, 22 insertions(+), 27 deletions(-)