| Message ID | 733408f48b1ed191f53518123ee6fc6d42287cc6.1618506910.git.christophe.leroy@csgroup.eu (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | Convert powerpc to GENERIC_PTDUMP | expand |
Hi Christophe, > Pagewalk ignores hugepd entries and walk down the tables > as if it was traditionnal entries, leading to crazy result. > > Add walk_hugepd_range() and use it to walk hugepage tables. > > Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu> > --- > mm/pagewalk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++------ > 1 file changed, 48 insertions(+), 6 deletions(-) > > diff --git a/mm/pagewalk.c b/mm/pagewalk.c > index e81640d9f177..410a9d8f7572 100644 > --- a/mm/pagewalk.c > +++ b/mm/pagewalk.c > @@ -58,6 +58,32 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, > return err; > } > > +static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr, > + unsigned long end, struct mm_walk *walk, int pdshift) > +{ > + int err = 0; > +#ifdef CONFIG_ARCH_HAS_HUGEPD > + const struct mm_walk_ops *ops = walk->ops; > + int shift = hugepd_shift(*phpd); > + int page_size = 1 << shift; > + > + if (addr & (page_size - 1)) > + return 0; > + > + for (;;) { > + pte_t *pte = hugepte_offset(*phpd, addr, pdshift); > + > + err = ops->pte_entry(pte, addr, addr + page_size, walk); > + if (err) > + break; > + if (addr >= end - page_size) > + break; > + addr += page_size; > + } Initially I thought this was a somewhat unintuitive way to structure this loop, but I see it parallels the structure of walk_pte_range_inner, so I think the consistency is worth it. I notice the pte walking code potentially takes some locks: does this code need to do that? arch/powerpc/mm/hugetlbpage.c says that hugepds are protected by the mm->page_table_lock, but I don't think we're taking it in this code. > +#endif > + return err; > +} > + > static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, > struct mm_walk *walk) > { > @@ -108,7 +134,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, > goto again; > } > > - err = walk_pte_range(pmd, addr, next, walk); > + if (is_hugepd(__hugepd(pmd_val(*pmd)))) > + err = walk_hugepd_range((hugepd_t *)pmd, addr, next, walk, PMD_SHIFT); > + else > + err = walk_pte_range(pmd, addr, next, walk); > if (err) > break; > } while (pmd++, addr = next, addr != end); > @@ -157,7 +186,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, > if (pud_none(*pud)) > goto again; > > - err = walk_pmd_range(pud, addr, next, walk); > + if (is_hugepd(__hugepd(pud_val(*pud)))) > + err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT); > + else > + err = walk_pmd_range(pud, addr, next, walk); I'm a bit worried you might end up calling into walk_hugepd_range with ops->pte_entry == NULL, and then jumping to 0. static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, struct mm_walk *walk) { ... pud = pud_offset(p4d, addr); do { ... if ((!walk->vma && (pud_leaf(*pud) || !pud_present(*pud))) || walk->action == ACTION_CONTINUE || !(ops->pmd_entry || ops->pte_entry)) <<< THIS CHECK continue; ... if (is_hugepd(__hugepd(pud_val(*pud)))) err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT); else err = walk_pmd_range(pud, addr, next, walk); if (err) break; } while (pud++, addr = next, addr != end); walk_pud_range will proceed if there is _either_ an ops->pmd_entry _or_ an ops->pte_entry, but walk_hugepd_range will call ops->pte_entry unconditionally. The same issue applies to walk_{p4d,pgd}_range... Kind regards, Daniel
Le 16/04/2021 à 00:43, Daniel Axtens a écrit : > Hi Christophe, > >> Pagewalk ignores hugepd entries and walk down the tables >> as if it was traditionnal entries, leading to crazy result. >> >> Add walk_hugepd_range() and use it to walk hugepage tables. >> >> Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu> >> --- >> mm/pagewalk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++------ >> 1 file changed, 48 insertions(+), 6 deletions(-) >> >> diff --git a/mm/pagewalk.c b/mm/pagewalk.c >> index e81640d9f177..410a9d8f7572 100644 >> --- a/mm/pagewalk.c >> +++ b/mm/pagewalk.c >> @@ -58,6 +58,32 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, >> return err; >> } >> >> +static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr, >> + unsigned long end, struct mm_walk *walk, int pdshift) >> +{ >> + int err = 0; >> +#ifdef CONFIG_ARCH_HAS_HUGEPD >> + const struct mm_walk_ops *ops = walk->ops; >> + int shift = hugepd_shift(*phpd); >> + int page_size = 1 << shift; >> + >> + if (addr & (page_size - 1)) >> + return 0; >> + >> + for (;;) { >> + pte_t *pte = hugepte_offset(*phpd, addr, pdshift); >> + >> + err = ops->pte_entry(pte, addr, addr + page_size, walk); >> + if (err) >> + break; >> + if (addr >= end - page_size) >> + break; >> + addr += page_size; >> + } > > Initially I thought this was a somewhat unintuitive way to structure > this loop, but I see it parallels the structure of walk_pte_range_inner, > so I think the consistency is worth it. > > I notice the pte walking code potentially takes some locks: does this > code need to do that? > > arch/powerpc/mm/hugetlbpage.c says that hugepds are protected by the > mm->page_table_lock, but I don't think we're taking it in this code. I'll add it, thanks. > >> +#endif >> + return err; >> +} >> + >> static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, >> struct mm_walk *walk) >> { >> @@ -108,7 +134,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, >> goto again; >> } >> >> - err = walk_pte_range(pmd, addr, next, walk); >> + if (is_hugepd(__hugepd(pmd_val(*pmd)))) >> + err = walk_hugepd_range((hugepd_t *)pmd, addr, next, walk, PMD_SHIFT); >> + else >> + err = walk_pte_range(pmd, addr, next, walk); >> if (err) >> break; >> } while (pmd++, addr = next, addr != end); >> @@ -157,7 +186,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, >> if (pud_none(*pud)) >> goto again; >> >> - err = walk_pmd_range(pud, addr, next, walk); >> + if (is_hugepd(__hugepd(pud_val(*pud)))) >> + err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT); >> + else >> + err = walk_pmd_range(pud, addr, next, walk); > > I'm a bit worried you might end up calling into walk_hugepd_range with > ops->pte_entry == NULL, and then jumping to 0. You are right, I missed it. I'll bail out of walk_hugepd_range() when ops->pte_entry is NULL. > > static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, > struct mm_walk *walk) > { > ... > pud = pud_offset(p4d, addr); > do { > ... > if ((!walk->vma && (pud_leaf(*pud) || !pud_present(*pud))) || > walk->action == ACTION_CONTINUE || > !(ops->pmd_entry || ops->pte_entry)) <<< THIS CHECK > continue; > ... > if (is_hugepd(__hugepd(pud_val(*pud)))) > err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT); > else > err = walk_pmd_range(pud, addr, next, walk); > if (err) > break; > } while (pud++, addr = next, addr != end); > > walk_pud_range will proceed if there is _either_ an ops->pmd_entry _or_ > an ops->pte_entry, but walk_hugepd_range will call ops->pte_entry > unconditionally. > > The same issue applies to walk_{p4d,pgd}_range... > > Kind regards, > Daniel > Thanks Christophe
diff --git a/mm/pagewalk.c b/mm/pagewalk.c index e81640d9f177..410a9d8f7572 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -58,6 +58,32 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, return err; } +static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr, + unsigned long end, struct mm_walk *walk, int pdshift) +{ + int err = 0; +#ifdef CONFIG_ARCH_HAS_HUGEPD + const struct mm_walk_ops *ops = walk->ops; + int shift = hugepd_shift(*phpd); + int page_size = 1 << shift; + + if (addr & (page_size - 1)) + return 0; + + for (;;) { + pte_t *pte = hugepte_offset(*phpd, addr, pdshift); + + err = ops->pte_entry(pte, addr, addr + page_size, walk); + if (err) + break; + if (addr >= end - page_size) + break; + addr += page_size; + } +#endif + return err; +} + static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -108,7 +134,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, goto again; } - err = walk_pte_range(pmd, addr, next, walk); + if (is_hugepd(__hugepd(pmd_val(*pmd)))) + err = walk_hugepd_range((hugepd_t *)pmd, addr, next, walk, PMD_SHIFT); + else + err = walk_pte_range(pmd, addr, next, walk); if (err) break; } while (pmd++, addr = next, addr != end); @@ -157,7 +186,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, if (pud_none(*pud)) goto again; - err = walk_pmd_range(pud, addr, next, walk); + if (is_hugepd(__hugepd(pud_val(*pud)))) + err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT); + else + err = walk_pmd_range(pud, addr, next, walk); if (err) break; } while (pud++, addr = next, addr != end); @@ -189,8 +221,13 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, if (err) break; } - if (ops->pud_entry || ops->pmd_entry || ops->pte_entry) - err = walk_pud_range(p4d, addr, next, walk); + if (ops->pud_entry || ops->pmd_entry || ops->pte_entry) { + if (is_hugepd(__hugepd(p4d_val(*p4d)))) + err = walk_hugepd_range((hugepd_t *)p4d, addr, next, walk, + P4D_SHIFT); + else + err = walk_pud_range(p4d, addr, next, walk); + } if (err) break; } while (p4d++, addr = next, addr != end); @@ -225,8 +262,13 @@ static int walk_pgd_range(unsigned long addr, unsigned long end, break; } if (ops->p4d_entry || ops->pud_entry || ops->pmd_entry || - ops->pte_entry) - err = walk_p4d_range(pgd, addr, next, walk); + ops->pte_entry) { + if (is_hugepd(__hugepd(pgd_val(*pgd)))) + err = walk_hugepd_range((hugepd_t *)pgd, addr, next, walk, + PGDIR_SHIFT); + else + err = walk_p4d_range(pgd, addr, next, walk); + } if (err) break; } while (pgd++, addr = next, addr != end);
Pagewalk ignores hugepd entries and walk down the tables as if it was traditionnal entries, leading to crazy result. Add walk_hugepd_range() and use it to walk hugepage tables. Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu> --- mm/pagewalk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-)