@@ -189,6 +189,12 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
pte_t *ptep;
ptep = pte_set_fixmap_offset(pmdp, addr);
+ if (likely(!(flags & BOOT_HEAD)))
+ ptep = pte_set_fixmap_offset(pmdp, addr);
+ else
+ /* for head.S, there is no __va() */
+ ptep = (pte_t *)__pmd_to_phys(*pmdp) + pte_index(addr);
+
do {
pte_t old_pte = READ_ONCE(*ptep);
@@ -204,7 +210,8 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
phys += PAGE_SIZE;
} while (ptep++, addr += PAGE_SIZE, addr != end);
- pte_clear_fixmap();
+ if (likely(!(flags & BOOT_HEAD)))
+ pte_clear_fixmap();
}
static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
@@ -253,7 +260,17 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
unsigned long next;
pmd_t *pmdp;
- pmdp = pmd_set_fixmap_offset(pudp, addr);
+ if (likely(!(flags & BOOT_HEAD))) {
+ pmdp = pmd_set_fixmap_offset(pudp, addr);
+ } else {
+#if CONFIG_PGTABLE_LEVELS > 2
+ /* for head.S, there is no __va() */
+ pmdp = (pmd_t *)__pud_to_phys(*pudp) + pmd_index(addr);
+#else
+ pmdp = (pmd_t *)pudp;
+#endif
+ }
+
do {
pmd_t old_pmd = READ_ONCE(*pmdp);
@@ -280,7 +297,8 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
phys += next - addr;
} while (pmdp++, addr = next, addr != end);
- pmd_clear_fixmap();
+ if (likely(!(flags & BOOT_HEAD)))
+ pmd_clear_fixmap();
}
static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
@@ -359,7 +377,17 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
}
SAFE_BUG_ON(flags, p4d_bad(p4d));
- pudp = pud_set_fixmap_offset(p4dp, addr);
+ if (likely(!(flags & BOOT_HEAD))) {
+ pudp = pud_set_fixmap_offset(p4dp, addr);
+ } else {
+#if CONFIG_PGTABLE_LEVELS > 3
+ /* for head.S, there is no __va() */
+ pudp = (pud_t *)__p4d_to_phys(*p4dp) + pud_index(addr);
+#else
+ pudp = (pud_t *)p4dp;
+#endif
+ }
+
do {
pud_t old_pud = READ_ONCE(*pudp);
@@ -388,7 +416,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
phys += next - addr;
} while (pudp++, addr = next, addr != end);
- pud_clear_fixmap();
+ if (likely(!(flags & BOOT_HEAD)))
+ pud_clear_fixmap();
}
static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
This patch is the last preparation for calling __create_pgd_mapping() from head.S. Under mmu-offset situation, __create_pgd_mapping() handles paddr of each pgtable level. All of pud_t */pmd_t */pte_t * points to paddr, and they should be handled carefully to avoid the involvement of __va(). Signed-off-by: Pingfan Liu <kernelfans@gmail.com> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Will Deacon <will@kernel.org> Cc: Ard Biesheuvel <ardb@kernel.org> Cc: Marc Zyngier <maz@kernel.org> Cc: Kristina Martsenko <kristina.martsenko@arm.com> Cc: James Morse <james.morse@arm.com> Cc: Steven Price <steven.price@arm.com> Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com> Cc: Pavel Tatashin <pasha.tatashin@soleen.com> Cc: Anshuman Khandual <anshuman.khandual@arm.com> Cc: Atish Patra <atish.patra@wdc.com> Cc: Mike Rapoport <rppt@kernel.org> Cc: Logan Gunthorpe <logang@deltatee.com> Cc: Mark Brown <broonie@kernel.org> To: linux-arm-kernel@lists.infradead.org --- arch/arm64/mm/mmu.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-)