diff mbox series

[v1,14/39] mm/rmap: introduce folio_add_anon_rmap_[pte|ptes|pmd]()

Message ID 20231211155652.131054-15-david@redhat.com (mailing list archive)
State New
Headers show
Series mm/rmap: interface overhaul | expand

Commit Message

David Hildenbrand Dec. 11, 2023, 3:56 p.m. UTC
Let's mimic what we did with folio_add_file_rmap_*() so we can similarly
replace page_add_anon_rmap() next.

Make the compiler always special-case on the granularity by using
__always_inline.

Note that the new functions ignore the RMAP_COMPOUND flag, which we will
remove as soon as page_add_anon_rmap() is gone.

Signed-off-by: David Hildenbrand <david@redhat.com>
---
 include/linux/rmap.h |   6 +++
 mm/rmap.c            | 118 ++++++++++++++++++++++++++++++-------------
 2 files changed, 88 insertions(+), 36 deletions(-)

Comments

Yin Fengwei Dec. 15, 2023, 2:26 a.m. UTC | #1
On 12/11/2023 11:56 PM, David Hildenbrand wrote:
> Let's mimic what we did with folio_add_file_rmap_*() so we can similarly
> replace page_add_anon_rmap() next.
> 
> Make the compiler always special-case on the granularity by using
> __always_inline.
> 
> Note that the new functions ignore the RMAP_COMPOUND flag, which we will
> remove as soon as page_add_anon_rmap() is gone.
> 
> Signed-off-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Yin Fengwei <fengwei.yin@intel.com>

With a small question below.

> ---
>   include/linux/rmap.h |   6 +++
>   mm/rmap.c            | 118 ++++++++++++++++++++++++++++++-------------
>   2 files changed, 88 insertions(+), 36 deletions(-)
> 
> diff --git a/include/linux/rmap.h b/include/linux/rmap.h
> index 7198905dc8be..3b5357cb1c09 100644
> --- a/include/linux/rmap.h
> +++ b/include/linux/rmap.h
> @@ -234,6 +234,12 @@ static inline void __folio_rmap_sanity_checks(struct folio *folio,
>    * rmap interfaces called when adding or removing pte of page
>    */
>   void folio_move_anon_rmap(struct folio *, struct vm_area_struct *);
> +void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages,
> +		struct vm_area_struct *, unsigned long address, rmap_t flags);
> +#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \
> +	folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags)
> +void folio_add_anon_rmap_pmd(struct folio *, struct page *,
> +		struct vm_area_struct *, unsigned long address, rmap_t flags);
>   void page_add_anon_rmap(struct page *, struct vm_area_struct *,
>   		unsigned long address, rmap_t flags);
>   void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
> diff --git a/mm/rmap.c b/mm/rmap.c
> index c5761986a411..7787499fa2ad 100644
> --- a/mm/rmap.c
> +++ b/mm/rmap.c
> @@ -1300,38 +1300,20 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
>   		unsigned long address, rmap_t flags)
>   {
>   	struct folio *folio = page_folio(page);
> -	atomic_t *mapped = &folio->_nr_pages_mapped;
> -	int nr = 0, nr_pmdmapped = 0;
> -	bool compound = flags & RMAP_COMPOUND;
> -	bool first;
>   
> -	/* Is page being mapped by PTE? Is this its first map to be added? */
> -	if (likely(!compound)) {
> -		first = atomic_inc_and_test(&page->_mapcount);
> -		nr = first;
> -		if (first && folio_test_large(folio)) {
> -			nr = atomic_inc_return_relaxed(mapped);
> -			nr = (nr < COMPOUND_MAPPED);
> -		}
> -	} else if (folio_test_pmd_mappable(folio)) {
> -		/* That test is redundant: it's for safety or to optimize out */
> +	if (likely(!(flags & RMAP_COMPOUND)))
> +		folio_add_anon_rmap_pte(folio, page, vma, address, flags);
> +	else
> +		folio_add_anon_rmap_pmd(folio, page, vma, address, flags);
> +}
>   
> -		first = atomic_inc_and_test(&folio->_entire_mapcount);
> -		if (first) {
> -			nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);
> -			if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {
> -				nr_pmdmapped = folio_nr_pages(folio);
> -				nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
> -				/* Raced ahead of a remove and another add? */
> -				if (unlikely(nr < 0))
> -					nr = 0;
> -			} else {
> -				/* Raced ahead of a remove of COMPOUND_MAPPED */
> -				nr = 0;
> -			}
> -		}
> -	}
> +static __always_inline void __folio_add_anon_rmap(struct folio *folio,
> +		struct page *page, int nr_pages, struct vm_area_struct *vma,
> +		unsigned long address, rmap_t flags, enum rmap_mode mode)
> +{
> +	unsigned int i, nr, nr_pmdmapped = 0;
>   
> +	nr = __folio_add_rmap(folio, page, nr_pages, mode, &nr_pmdmapped);
>   	if (nr_pmdmapped)
>   		__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr_pmdmapped);
>   	if (nr)
> @@ -1345,18 +1327,34 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
>   		 * folio->index right when not given the address of the head
>   		 * page.
>   		 */
> -		VM_WARN_ON_FOLIO(folio_test_large(folio) && !compound, folio);
> +		VM_WARN_ON_FOLIO(folio_test_large(folio) &&
> +				 mode != RMAP_MODE_PMD, folio);
>   		__folio_set_anon(folio, vma, address,
>   				 !!(flags & RMAP_EXCLUSIVE));
>   	} else if (likely(!folio_test_ksm(folio))) {
>   		__page_check_anon_rmap(folio, page, vma, address);
>   	}
> -	if (flags & RMAP_EXCLUSIVE)
> -		SetPageAnonExclusive(page);
> -	/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
> -	VM_WARN_ON_FOLIO((atomic_read(&page->_mapcount) > 0 ||
> -			  (folio_test_large(folio) && folio_entire_mapcount(folio) > 1)) &&
> -			 PageAnonExclusive(page), folio);
> +
> +	if (flags & RMAP_EXCLUSIVE) {
> +		switch (mode) {
> +		case RMAP_MODE_PTE:
> +			for (i = 0; i < nr_pages; i++)
> +				SetPageAnonExclusive(page + i);
> +			break;
> +		case RMAP_MODE_PMD:
> +			SetPageAnonExclusive(page);
> +			break;
> +		}
> +	}
> +	for (i = 0; i < nr_pages; i++) {
> +		struct page *cur_page = page + i;
> +
> +		/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
> +		VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||
> +				  (folio_test_large(folio) &&
> +				   folio_entire_mapcount(folio) > 1)) &&
> +				 PageAnonExclusive(cur_page), folio);
> +	}
This change will iterate all pages for PMD case. The original behavior
didn't check all pages. Is this change by purpose? Thanks.

>   
>   	/*
>   	 * For large folio, only mlock it if it's fully mapped to VMA. It's
> @@ -1368,6 +1366,54 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
>   		mlock_vma_folio(folio, vma);
>   }
>   
> +/**
> + * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
> + * @folio:	The folio to add the mappings to
> + * @page:	The first page to add
> + * @nr_pages:	The number of pages which will be mapped
> + * @vma:	The vm area in which the mappings are added
> + * @address:	The user virtual address of the first page to map
> + * @flags:	The rmap flags
> + *
> + * The page range of folio is defined by [first_page, first_page + nr_pages)
> + *
> + * The caller needs to hold the page table lock, and the page must be locked in
> + * the anon_vma case: to serialize mapping,index checking after setting,
> + * and to ensure that an anon folio is not being upgraded racily to a KSM folio
> + * (but KSM folios are never downgraded).
> + */
> +void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
> +		int nr_pages, struct vm_area_struct *vma, unsigned long address,
> +		rmap_t flags)
> +{
> +	__folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
> +			      RMAP_MODE_PTE);
> +}
> +
> +/**
> + * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
> + * @folio:	The folio to add the mapping to
> + * @page:	The first page to add
> + * @vma:	The vm area in which the mapping is added
> + * @address:	The user virtual address of the first page to map
> + * @flags:	The rmap flags
> + *
> + * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
> + *
> + * The caller needs to hold the page table lock, and the page must be locked in
> + * the anon_vma case: to serialize mapping,index checking after setting.
> + */
> +void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
> +		struct vm_area_struct *vma, unsigned long address, rmap_t flags)
> +{
> +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> +	__folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
> +			      RMAP_MODE_PMD);
> +#else
> +	WARN_ON_ONCE(true);
> +#endif
> +}
> +
>   /**
>    * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
>    * @folio:	The folio to add the mapping to.
David Hildenbrand Dec. 15, 2023, 3:16 p.m. UTC | #2
On 15.12.23 03:26, Yin, Fengwei wrote:
> 
> 
> On 12/11/2023 11:56 PM, David Hildenbrand wrote:
>> Let's mimic what we did with folio_add_file_rmap_*() so we can similarly
>> replace page_add_anon_rmap() next.
>>
>> Make the compiler always special-case on the granularity by using
>> __always_inline.
>>
>> Note that the new functions ignore the RMAP_COMPOUND flag, which we will
>> remove as soon as page_add_anon_rmap() is gone.
>>
>> Signed-off-by: David Hildenbrand <david@redhat.com>
> Reviewed-by: Yin Fengwei <fengwei.yin@intel.com>
> 
> With a small question below.
> 

Thanks!

[...]

>> +	if (flags & RMAP_EXCLUSIVE) {
>> +		switch (mode) {
>> +		case RMAP_MODE_PTE:
>> +			for (i = 0; i < nr_pages; i++)
>> +				SetPageAnonExclusive(page + i);
>> +			break;
>> +		case RMAP_MODE_PMD:
>> +			SetPageAnonExclusive(page);
>> +			break;
>> +		}
>> +	}
>> +	for (i = 0; i < nr_pages; i++) {
>> +		struct page *cur_page = page + i;
>> +
>> +		/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
>> +		VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||
>> +				  (folio_test_large(folio) &&
>> +				   folio_entire_mapcount(folio) > 1)) &&
>> +				 PageAnonExclusive(cur_page), folio);
>> +	}
> This change will iterate all pages for PMD case. The original behavior
> didn't check all pages. Is this change by purpose? Thanks.

Yes, on purpose. I first thought about also separating the code paths 
here, but realized that it makes much more sense to check each 
individual subpage that is effectively getting mapped by that PMD, 
instead of only the head page.

I'll add a comment to the patch description.
Ryan Roberts Dec. 18, 2023, 4:26 p.m. UTC | #3
On 11/12/2023 15:56, David Hildenbrand wrote:
> Let's mimic what we did with folio_add_file_rmap_*() so we can similarly
> replace page_add_anon_rmap() next.
> 
> Make the compiler always special-case on the granularity by using
> __always_inline.
> 
> Note that the new functions ignore the RMAP_COMPOUND flag, which we will
> remove as soon as page_add_anon_rmap() is gone.
> 
> Signed-off-by: David Hildenbrand <david@redhat.com>
> ---
>  include/linux/rmap.h |   6 +++
>  mm/rmap.c            | 118 ++++++++++++++++++++++++++++++-------------
>  2 files changed, 88 insertions(+), 36 deletions(-)
> 
> diff --git a/include/linux/rmap.h b/include/linux/rmap.h
> index 7198905dc8be..3b5357cb1c09 100644
> --- a/include/linux/rmap.h
> +++ b/include/linux/rmap.h
> @@ -234,6 +234,12 @@ static inline void __folio_rmap_sanity_checks(struct folio *folio,
>   * rmap interfaces called when adding or removing pte of page
>   */
>  void folio_move_anon_rmap(struct folio *, struct vm_area_struct *);
> +void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages,
> +		struct vm_area_struct *, unsigned long address, rmap_t flags);
> +#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \
> +	folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags)
> +void folio_add_anon_rmap_pmd(struct folio *, struct page *,
> +		struct vm_area_struct *, unsigned long address, rmap_t flags);
>  void page_add_anon_rmap(struct page *, struct vm_area_struct *,
>  		unsigned long address, rmap_t flags);
>  void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
> diff --git a/mm/rmap.c b/mm/rmap.c
> index c5761986a411..7787499fa2ad 100644
> --- a/mm/rmap.c
> +++ b/mm/rmap.c
> @@ -1300,38 +1300,20 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
>  		unsigned long address, rmap_t flags)
>  {
>  	struct folio *folio = page_folio(page);
> -	atomic_t *mapped = &folio->_nr_pages_mapped;
> -	int nr = 0, nr_pmdmapped = 0;
> -	bool compound = flags & RMAP_COMPOUND;
> -	bool first;
>  
> -	/* Is page being mapped by PTE? Is this its first map to be added? */
> -	if (likely(!compound)) {
> -		first = atomic_inc_and_test(&page->_mapcount);
> -		nr = first;
> -		if (first && folio_test_large(folio)) {
> -			nr = atomic_inc_return_relaxed(mapped);
> -			nr = (nr < COMPOUND_MAPPED);
> -		}
> -	} else if (folio_test_pmd_mappable(folio)) {
> -		/* That test is redundant: it's for safety or to optimize out */
> +	if (likely(!(flags & RMAP_COMPOUND)))
> +		folio_add_anon_rmap_pte(folio, page, vma, address, flags);
> +	else
> +		folio_add_anon_rmap_pmd(folio, page, vma, address, flags);
> +}
>  
> -		first = atomic_inc_and_test(&folio->_entire_mapcount);
> -		if (first) {
> -			nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);
> -			if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {
> -				nr_pmdmapped = folio_nr_pages(folio);
> -				nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
> -				/* Raced ahead of a remove and another add? */
> -				if (unlikely(nr < 0))
> -					nr = 0;
> -			} else {
> -				/* Raced ahead of a remove of COMPOUND_MAPPED */
> -				nr = 0;
> -			}
> -		}
> -	}
> +static __always_inline void __folio_add_anon_rmap(struct folio *folio,
> +		struct page *page, int nr_pages, struct vm_area_struct *vma,
> +		unsigned long address, rmap_t flags, enum rmap_mode mode)
> +{
> +	unsigned int i, nr, nr_pmdmapped = 0;
>  
> +	nr = __folio_add_rmap(folio, page, nr_pages, mode, &nr_pmdmapped);
>  	if (nr_pmdmapped)
>  		__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr_pmdmapped);
>  	if (nr)
> @@ -1345,18 +1327,34 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
>  		 * folio->index right when not given the address of the head
>  		 * page.
>  		 */
> -		VM_WARN_ON_FOLIO(folio_test_large(folio) && !compound, folio);
> +		VM_WARN_ON_FOLIO(folio_test_large(folio) &&
> +				 mode != RMAP_MODE_PMD, folio);
>  		__folio_set_anon(folio, vma, address,
>  				 !!(flags & RMAP_EXCLUSIVE));
>  	} else if (likely(!folio_test_ksm(folio))) {
>  		__page_check_anon_rmap(folio, page, vma, address);
>  	}
> -	if (flags & RMAP_EXCLUSIVE)
> -		SetPageAnonExclusive(page);
> -	/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
> -	VM_WARN_ON_FOLIO((atomic_read(&page->_mapcount) > 0 ||
> -			  (folio_test_large(folio) && folio_entire_mapcount(folio) > 1)) &&
> -			 PageAnonExclusive(page), folio);
> +
> +	if (flags & RMAP_EXCLUSIVE) {
> +		switch (mode) {
> +		case RMAP_MODE_PTE:
> +			for (i = 0; i < nr_pages; i++)
> +				SetPageAnonExclusive(page + i);
> +			break;
> +		case RMAP_MODE_PMD:
> +			SetPageAnonExclusive(page);

Just to check; I suppose only setting this on the head is ok, because it's an
exclusive mapping and therefore by definition it can only be mapped by pmd?

> +			break;
> +		}
> +	}
> +	for (i = 0; i < nr_pages; i++) {
> +		struct page *cur_page = page + i;
> +
> +		/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
> +		VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||
> +				  (folio_test_large(folio) &&
> +				   folio_entire_mapcount(folio) > 1)) &&
> +				 PageAnonExclusive(cur_page), folio);
> +	}
>  
>  	/*
>  	 * For large folio, only mlock it if it's fully mapped to VMA. It's
> @@ -1368,6 +1366,54 @@ void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
>  		mlock_vma_folio(folio, vma);
>  }
>  
> +/**
> + * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
> + * @folio:	The folio to add the mappings to
> + * @page:	The first page to add
> + * @nr_pages:	The number of pages which will be mapped
> + * @vma:	The vm area in which the mappings are added
> + * @address:	The user virtual address of the first page to map
> + * @flags:	The rmap flags
> + *
> + * The page range of folio is defined by [first_page, first_page + nr_pages)
> + *
> + * The caller needs to hold the page table lock, and the page must be locked in
> + * the anon_vma case: to serialize mapping,index checking after setting,
> + * and to ensure that an anon folio is not being upgraded racily to a KSM folio
> + * (but KSM folios are never downgraded).
> + */
> +void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
> +		int nr_pages, struct vm_area_struct *vma, unsigned long address,
> +		rmap_t flags)
> +{
> +	__folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
> +			      RMAP_MODE_PTE);
> +}
> +
> +/**
> + * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
> + * @folio:	The folio to add the mapping to
> + * @page:	The first page to add
> + * @vma:	The vm area in which the mapping is added
> + * @address:	The user virtual address of the first page to map
> + * @flags:	The rmap flags
> + *
> + * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
> + *
> + * The caller needs to hold the page table lock, and the page must be locked in
> + * the anon_vma case: to serialize mapping,index checking after setting.
> + */
> +void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
> +		struct vm_area_struct *vma, unsigned long address, rmap_t flags)
> +{
> +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> +	__folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
> +			      RMAP_MODE_PMD);
> +#else
> +	WARN_ON_ONCE(true);
> +#endif
> +}
> +
>  /**
>   * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
>   * @folio:	The folio to add the mapping to.
David Hildenbrand Dec. 18, 2023, 5:02 p.m. UTC | #4
>> -	if (flags & RMAP_EXCLUSIVE)
>> -		SetPageAnonExclusive(page);
>> -	/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
>> -	VM_WARN_ON_FOLIO((atomic_read(&page->_mapcount) > 0 ||
>> -			  (folio_test_large(folio) && folio_entire_mapcount(folio) > 1)) &&
>> -			 PageAnonExclusive(page), folio);
>> +
>> +	if (flags & RMAP_EXCLUSIVE) {
>> +		switch (mode) {
>> +		case RMAP_MODE_PTE:
>> +			for (i = 0; i < nr_pages; i++)
>> +				SetPageAnonExclusive(page + i);
>> +			break;
>> +		case RMAP_MODE_PMD:
>> +			SetPageAnonExclusive(page);
> 
> Just to check; I suppose only setting this on the head is ok, because it's an
> exclusive mapping and therefore by definition it can only be mapped by pmd?

Yes. And when PTE-remapping, we will push the flag to all tail pages. No 
change in behavior :)
diff mbox series

Patch

diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 7198905dc8be..3b5357cb1c09 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -234,6 +234,12 @@  static inline void __folio_rmap_sanity_checks(struct folio *folio,
  * rmap interfaces called when adding or removing pte of page
  */
 void folio_move_anon_rmap(struct folio *, struct vm_area_struct *);
+void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages,
+		struct vm_area_struct *, unsigned long address, rmap_t flags);
+#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \
+	folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags)
+void folio_add_anon_rmap_pmd(struct folio *, struct page *,
+		struct vm_area_struct *, unsigned long address, rmap_t flags);
 void page_add_anon_rmap(struct page *, struct vm_area_struct *,
 		unsigned long address, rmap_t flags);
 void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
diff --git a/mm/rmap.c b/mm/rmap.c
index c5761986a411..7787499fa2ad 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1300,38 +1300,20 @@  void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
 		unsigned long address, rmap_t flags)
 {
 	struct folio *folio = page_folio(page);
-	atomic_t *mapped = &folio->_nr_pages_mapped;
-	int nr = 0, nr_pmdmapped = 0;
-	bool compound = flags & RMAP_COMPOUND;
-	bool first;
 
-	/* Is page being mapped by PTE? Is this its first map to be added? */
-	if (likely(!compound)) {
-		first = atomic_inc_and_test(&page->_mapcount);
-		nr = first;
-		if (first && folio_test_large(folio)) {
-			nr = atomic_inc_return_relaxed(mapped);
-			nr = (nr < COMPOUND_MAPPED);
-		}
-	} else if (folio_test_pmd_mappable(folio)) {
-		/* That test is redundant: it's for safety or to optimize out */
+	if (likely(!(flags & RMAP_COMPOUND)))
+		folio_add_anon_rmap_pte(folio, page, vma, address, flags);
+	else
+		folio_add_anon_rmap_pmd(folio, page, vma, address, flags);
+}
 
-		first = atomic_inc_and_test(&folio->_entire_mapcount);
-		if (first) {
-			nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);
-			if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {
-				nr_pmdmapped = folio_nr_pages(folio);
-				nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
-				/* Raced ahead of a remove and another add? */
-				if (unlikely(nr < 0))
-					nr = 0;
-			} else {
-				/* Raced ahead of a remove of COMPOUND_MAPPED */
-				nr = 0;
-			}
-		}
-	}
+static __always_inline void __folio_add_anon_rmap(struct folio *folio,
+		struct page *page, int nr_pages, struct vm_area_struct *vma,
+		unsigned long address, rmap_t flags, enum rmap_mode mode)
+{
+	unsigned int i, nr, nr_pmdmapped = 0;
 
+	nr = __folio_add_rmap(folio, page, nr_pages, mode, &nr_pmdmapped);
 	if (nr_pmdmapped)
 		__lruvec_stat_mod_folio(folio, NR_ANON_THPS, nr_pmdmapped);
 	if (nr)
@@ -1345,18 +1327,34 @@  void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
 		 * folio->index right when not given the address of the head
 		 * page.
 		 */
-		VM_WARN_ON_FOLIO(folio_test_large(folio) && !compound, folio);
+		VM_WARN_ON_FOLIO(folio_test_large(folio) &&
+				 mode != RMAP_MODE_PMD, folio);
 		__folio_set_anon(folio, vma, address,
 				 !!(flags & RMAP_EXCLUSIVE));
 	} else if (likely(!folio_test_ksm(folio))) {
 		__page_check_anon_rmap(folio, page, vma, address);
 	}
-	if (flags & RMAP_EXCLUSIVE)
-		SetPageAnonExclusive(page);
-	/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
-	VM_WARN_ON_FOLIO((atomic_read(&page->_mapcount) > 0 ||
-			  (folio_test_large(folio) && folio_entire_mapcount(folio) > 1)) &&
-			 PageAnonExclusive(page), folio);
+
+	if (flags & RMAP_EXCLUSIVE) {
+		switch (mode) {
+		case RMAP_MODE_PTE:
+			for (i = 0; i < nr_pages; i++)
+				SetPageAnonExclusive(page + i);
+			break;
+		case RMAP_MODE_PMD:
+			SetPageAnonExclusive(page);
+			break;
+		}
+	}
+	for (i = 0; i < nr_pages; i++) {
+		struct page *cur_page = page + i;
+
+		/* While PTE-mapping a THP we have a PMD and a PTE mapping. */
+		VM_WARN_ON_FOLIO((atomic_read(&cur_page->_mapcount) > 0 ||
+				  (folio_test_large(folio) &&
+				   folio_entire_mapcount(folio) > 1)) &&
+				 PageAnonExclusive(cur_page), folio);
+	}
 
 	/*
 	 * For large folio, only mlock it if it's fully mapped to VMA. It's
@@ -1368,6 +1366,54 @@  void page_add_anon_rmap(struct page *page, struct vm_area_struct *vma,
 		mlock_vma_folio(folio, vma);
 }
 
+/**
+ * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
+ * @folio:	The folio to add the mappings to
+ * @page:	The first page to add
+ * @nr_pages:	The number of pages which will be mapped
+ * @vma:	The vm area in which the mappings are added
+ * @address:	The user virtual address of the first page to map
+ * @flags:	The rmap flags
+ *
+ * The page range of folio is defined by [first_page, first_page + nr_pages)
+ *
+ * The caller needs to hold the page table lock, and the page must be locked in
+ * the anon_vma case: to serialize mapping,index checking after setting,
+ * and to ensure that an anon folio is not being upgraded racily to a KSM folio
+ * (but KSM folios are never downgraded).
+ */
+void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
+		int nr_pages, struct vm_area_struct *vma, unsigned long address,
+		rmap_t flags)
+{
+	__folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
+			      RMAP_MODE_PTE);
+}
+
+/**
+ * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
+ * @folio:	The folio to add the mapping to
+ * @page:	The first page to add
+ * @vma:	The vm area in which the mapping is added
+ * @address:	The user virtual address of the first page to map
+ * @flags:	The rmap flags
+ *
+ * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
+ *
+ * The caller needs to hold the page table lock, and the page must be locked in
+ * the anon_vma case: to serialize mapping,index checking after setting.
+ */
+void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
+		struct vm_area_struct *vma, unsigned long address, rmap_t flags)
+{
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	__folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
+			      RMAP_MODE_PMD);
+#else
+	WARN_ON_ONCE(true);
+#endif
+}
+
 /**
  * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
  * @folio:	The folio to add the mapping to.