| Message ID | 20210204035043.36609-8-songmuchun@bytedance.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Free some vmemmap pages of HugeTLB page | expand |
On 2021/2/4 11:50, Muchun Song wrote: > For HugeTLB page, there are more metadata to save in the struct page. > But the head struct page cannot meet our needs, so we have to abuse > other tail struct page to store the metadata. In order to avoid > conflicts caused by subsequent use of more tail struct pages, we can > gather these discrete indexes of tail struct page. In this case, it > will be easier to add a new tail page index later. > > There are only (RESERVE_VMEMMAP_SIZE / sizeof(struct page)) struct > page structs that can be used when CONFIG_HUGETLB_PAGE_FREE_VMEMMAP, > so add a BUILD_BUG_ON to catch invalid usage of the tail struct page. > > Signed-off-by: Muchun Song <songmuchun@bytedance.com> > Reviewed-by: Oscar Salvador <osalvador@suse.de> Thanks. Reviewed-by: Miaohe Lin <linmiaohe@huawei.com> > --- > include/linux/hugetlb.h | 20 ++++++++++++++++++-- > include/linux/hugetlb_cgroup.h | 19 +++++++++++-------- > mm/hugetlb_vmemmap.c | 8 ++++++++ > 3 files changed, 37 insertions(+), 10 deletions(-) > > diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h > index 775aea53669a..822ab2f5542a 100644 > --- a/include/linux/hugetlb.h > +++ b/include/linux/hugetlb.h > @@ -28,6 +28,22 @@ typedef struct { unsigned long pd; } hugepd_t; > #include <linux/shm.h> > #include <asm/tlbflush.h> > > +/* > + * For HugeTLB page, there are more metadata to save in the struct page. But > + * the head struct page cannot meet our needs, so we have to abuse other tail > + * struct page to store the metadata. In order to avoid conflicts caused by > + * subsequent use of more tail struct pages, we gather these discrete indexes > + * of tail struct page here. > + */ > +enum { > + SUBPAGE_INDEX_SUBPOOL = 1, /* reuse page->private */ > +#ifdef CONFIG_CGROUP_HUGETLB > + SUBPAGE_INDEX_CGROUP, /* reuse page->private */ > + SUBPAGE_INDEX_CGROUP_RSVD, /* reuse page->private */ > +#endif > + NR_USED_SUBPAGE, > +}; > + > struct hugepage_subpool { > spinlock_t lock; > long count; > @@ -607,13 +623,13 @@ extern unsigned int default_hstate_idx; > */ > static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage) > { > - return (struct hugepage_subpool *)(hpage+1)->private; > + return (void *)page_private(hpage + SUBPAGE_INDEX_SUBPOOL); > } > > static inline void hugetlb_set_page_subpool(struct page *hpage, > struct hugepage_subpool *subpool) > { > - set_page_private(hpage+1, (unsigned long)subpool); > + set_page_private(hpage + SUBPAGE_INDEX_SUBPOOL, (unsigned long)subpool); > } > > static inline struct hstate *hstate_file(struct file *f) > diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h > index 2ad6e92f124a..c0cae6a704f2 100644 > --- a/include/linux/hugetlb_cgroup.h > +++ b/include/linux/hugetlb_cgroup.h > @@ -21,15 +21,16 @@ struct hugetlb_cgroup; > struct resv_map; > struct file_region; > > +#ifdef CONFIG_CGROUP_HUGETLB > /* > * Minimum page order trackable by hugetlb cgroup. > * At least 4 pages are necessary for all the tracking information. > - * The second tail page (hpage[2]) is the fault usage cgroup. > - * The third tail page (hpage[3]) is the reservation usage cgroup. > + * The second tail page (hpage[SUBPAGE_INDEX_CGROUP]) is the fault > + * usage cgroup. The third tail page (hpage[SUBPAGE_INDEX_CGROUP_RSVD]) > + * is the reservation usage cgroup. > */ > -#define HUGETLB_CGROUP_MIN_ORDER 2 > +#define HUGETLB_CGROUP_MIN_ORDER order_base_2(NR_USED_SUBPAGE) > > -#ifdef CONFIG_CGROUP_HUGETLB > enum hugetlb_memory_event { > HUGETLB_MAX, > HUGETLB_NR_MEMORY_EVENTS, > @@ -66,9 +67,9 @@ __hugetlb_cgroup_from_page(struct page *page, bool rsvd) > if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) > return NULL; > if (rsvd) > - return (struct hugetlb_cgroup *)page[3].private; > + return (void *)page_private(page + SUBPAGE_INDEX_CGROUP_RSVD); > else > - return (struct hugetlb_cgroup *)page[2].private; > + return (void *)page_private(page + SUBPAGE_INDEX_CGROUP); > } > > static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) > @@ -90,9 +91,11 @@ static inline int __set_hugetlb_cgroup(struct page *page, > if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) > return -1; > if (rsvd) > - page[3].private = (unsigned long)h_cg; > + set_page_private(page + SUBPAGE_INDEX_CGROUP_RSVD, > + (unsigned long)h_cg); > else > - page[2].private = (unsigned long)h_cg; > + set_page_private(page + SUBPAGE_INDEX_CGROUP, > + (unsigned long)h_cg); > return 0; > } > > diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c > index 36ebd677e606..8efad9978821 100644 > --- a/mm/hugetlb_vmemmap.c > +++ b/mm/hugetlb_vmemmap.c > @@ -272,6 +272,14 @@ void __init hugetlb_vmemmap_init(struct hstate *h) > unsigned int nr_pages = pages_per_huge_page(h); > unsigned int vmemmap_pages; > > + /* > + * There are only (RESERVE_VMEMMAP_SIZE / sizeof(struct page)) struct > + * page structs that can be used when CONFIG_HUGETLB_PAGE_FREE_VMEMMAP, > + * so add a BUILD_BUG_ON to catch invalid usage of the tail struct page. > + */ > + BUILD_BUG_ON(NR_USED_SUBPAGE >= > + RESERVE_VMEMMAP_SIZE / sizeof(struct page)); > + > if (!hugetlb_free_vmemmap_enabled) > return; > >
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 775aea53669a..822ab2f5542a 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -28,6 +28,22 @@ typedef struct { unsigned long pd; } hugepd_t; #include <linux/shm.h> #include <asm/tlbflush.h> +/* + * For HugeTLB page, there are more metadata to save in the struct page. But + * the head struct page cannot meet our needs, so we have to abuse other tail + * struct page to store the metadata. In order to avoid conflicts caused by + * subsequent use of more tail struct pages, we gather these discrete indexes + * of tail struct page here. + */ +enum { + SUBPAGE_INDEX_SUBPOOL = 1, /* reuse page->private */ +#ifdef CONFIG_CGROUP_HUGETLB + SUBPAGE_INDEX_CGROUP, /* reuse page->private */ + SUBPAGE_INDEX_CGROUP_RSVD, /* reuse page->private */ +#endif + NR_USED_SUBPAGE, +}; + struct hugepage_subpool { spinlock_t lock; long count; @@ -607,13 +623,13 @@ extern unsigned int default_hstate_idx; */ static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage) { - return (struct hugepage_subpool *)(hpage+1)->private; + return (void *)page_private(hpage + SUBPAGE_INDEX_SUBPOOL); } static inline void hugetlb_set_page_subpool(struct page *hpage, struct hugepage_subpool *subpool) { - set_page_private(hpage+1, (unsigned long)subpool); + set_page_private(hpage + SUBPAGE_INDEX_SUBPOOL, (unsigned long)subpool); } static inline struct hstate *hstate_file(struct file *f) diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index 2ad6e92f124a..c0cae6a704f2 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -21,15 +21,16 @@ struct hugetlb_cgroup; struct resv_map; struct file_region; +#ifdef CONFIG_CGROUP_HUGETLB /* * Minimum page order trackable by hugetlb cgroup. * At least 4 pages are necessary for all the tracking information. - * The second tail page (hpage[2]) is the fault usage cgroup. - * The third tail page (hpage[3]) is the reservation usage cgroup. + * The second tail page (hpage[SUBPAGE_INDEX_CGROUP]) is the fault + * usage cgroup. The third tail page (hpage[SUBPAGE_INDEX_CGROUP_RSVD]) + * is the reservation usage cgroup. */ -#define HUGETLB_CGROUP_MIN_ORDER 2 +#define HUGETLB_CGROUP_MIN_ORDER order_base_2(NR_USED_SUBPAGE) -#ifdef CONFIG_CGROUP_HUGETLB enum hugetlb_memory_event { HUGETLB_MAX, HUGETLB_NR_MEMORY_EVENTS, @@ -66,9 +67,9 @@ __hugetlb_cgroup_from_page(struct page *page, bool rsvd) if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) return NULL; if (rsvd) - return (struct hugetlb_cgroup *)page[3].private; + return (void *)page_private(page + SUBPAGE_INDEX_CGROUP_RSVD); else - return (struct hugetlb_cgroup *)page[2].private; + return (void *)page_private(page + SUBPAGE_INDEX_CGROUP); } static inline struct hugetlb_cgroup *hugetlb_cgroup_from_page(struct page *page) @@ -90,9 +91,11 @@ static inline int __set_hugetlb_cgroup(struct page *page, if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) return -1; if (rsvd) - page[3].private = (unsigned long)h_cg; + set_page_private(page + SUBPAGE_INDEX_CGROUP_RSVD, + (unsigned long)h_cg); else - page[2].private = (unsigned long)h_cg; + set_page_private(page + SUBPAGE_INDEX_CGROUP, + (unsigned long)h_cg); return 0; } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 36ebd677e606..8efad9978821 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -272,6 +272,14 @@ void __init hugetlb_vmemmap_init(struct hstate *h) unsigned int nr_pages = pages_per_huge_page(h); unsigned int vmemmap_pages; + /* + * There are only (RESERVE_VMEMMAP_SIZE / sizeof(struct page)) struct + * page structs that can be used when CONFIG_HUGETLB_PAGE_FREE_VMEMMAP, + * so add a BUILD_BUG_ON to catch invalid usage of the tail struct page. + */ + BUILD_BUG_ON(NR_USED_SUBPAGE >= + RESERVE_VMEMMAP_SIZE / sizeof(struct page)); + if (!hugetlb_free_vmemmap_enabled) return;