@@ -59,13 +59,19 @@ enum fixed_addresses {
#endif /* CONFIG_ACPI_APEI_GHES */
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define TRAMP_TEXT_SIZE (PAGE_SIZE_MIN * 3)
#ifdef CONFIG_RELOCATABLE
- FIX_ENTRY_TRAMP_TEXT4, /* one extra slot for the data page */
+#define TRAMP_DATA_SIZE PAGE_SIZE_MIN
+#define TRAMP_PAD_SIZE (PAGE_SIZE_MAX - PAGE_SIZE_MIN)
+#else
+#define TRAMP_DATA_SIZE 0
+#define TRAMP_PAD_SIZE 0
#endif
- FIX_ENTRY_TRAMP_TEXT3,
- FIX_ENTRY_TRAMP_TEXT2,
- FIX_ENTRY_TRAMP_TEXT1,
-#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))
+#define TRAMP_SIZE (TRAMP_TEXT_SIZE + TRAMP_DATA_SIZE + TRAMP_PAD_SIZE)
+ FIX_ENTRY_TRAMP_END,
+ FIX_ENTRY_TRAMP_BEGIN = FIX_ENTRY_TRAMP_END +
+ DIV_ROUND_UP(TRAMP_SIZE, PAGE_SIZE_MIN) - 1,
+#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_BEGIN))
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
__end_of_permanent_fixed_addresses,
@@ -21,6 +21,7 @@ extern char __exittext_begin[], __exittext_end[];
extern char __irqentry_text_start[], __irqentry_text_end[];
extern char __mmuoff_data_start[], __mmuoff_data_end[];
extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
+extern char __entry_tramp_rodata_start[], __entry_tramp_rodata_end[];
extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[];
static inline size_t entry_tramp_text_size(void)
@@ -118,7 +118,9 @@ jiffies = jiffies_64;
*(.entry.tramp.text) \
__entry_tramp_text_end = .; \
. = ALIGN(PAGE_SIZE_MAX); \
- *(.entry.tramp.rodata)
+ __entry_tramp_rodata_start = .; \
+ *(.entry.tramp.rodata) \
+ __entry_tramp_rodata_end = .;
#else
#define TRAMP_TEXT
#endif
@@ -734,25 +734,31 @@ static int __init map_entry_trampoline(void)
return 0;
pgprot_t prot = kernel_exec_prot();
- phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
+ phys_addr_t pa_text = __pa_symbol(__entry_tramp_text_start);
+ phys_addr_t pa_data = __pa_symbol(__entry_tramp_rodata_start);
+ int slot = FIX_ENTRY_TRAMP_BEGIN;
/* The trampoline is always mapped and can therefore be global */
pgprot_val(prot) &= ~PTE_NG;
/* Map only the text into the trampoline page table */
memset(tramp_pg_dir, 0, PGD_SIZE);
- __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
+ __create_pgd_mapping(tramp_pg_dir, pa_text, TRAMP_VALIAS,
entry_tramp_text_size(), prot,
__pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
/* Map both the text and data into the kernel page table */
- for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
- pa_start + i * PAGE_SIZE, prot);
+ for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) {
+ __set_fixmap(slot, pa_text, prot);
+ pa_text += PAGE_SIZE;
+ slot--;
+ }
- if (IS_ENABLED(CONFIG_RELOCATABLE))
- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
- pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO);
+ if (IS_ENABLED(CONFIG_RELOCATABLE)) {
+ slot -= (pa_data - pa_text) / PAGE_SIZE;
+ VM_BUG_ON(slot < FIX_ENTRY_TRAMP_END);
+ __set_fixmap(slot, pa_data, PAGE_KERNEL_RO);
+ }
return 0;
}
Now that the trampoline rodata is aligned to the next PAGE_SIZE_MAX boundary after the end of the trampoline text, the code that maps it in the fixmap is incorrect, because it still assumes the rodata is in the next page immediately after the text. Of course it still works for now with compile-time page size but for boot-time page size when selecting a page size less than PAGE_SIZE_MAX, it will fail. So let's fix that by allocating sufficient fixmap slots to cover the extra alignment padding in the worst case (PAGE_SIZE == PAGE_SIZE_MIN) and explicitly mapping the rodata to the slot offset correctly from the text. Signed-off-by: Ryan Roberts <ryan.roberts@arm.com> --- ***NOTE*** Any confused maintainers may want to read the cover note here for context: https://lore.kernel.org/all/20241014105514.3206191-1-ryan.roberts@arm.com/ arch/arm64/include/asm/fixmap.h | 16 +++++++++++----- arch/arm64/include/asm/sections.h | 1 + arch/arm64/kernel/vmlinux.lds.S | 4 +++- arch/arm64/mm/mmu.c | 22 ++++++++++++++-------- 4 files changed, 29 insertions(+), 14 deletions(-)